2024-09-14T06:37:46.475 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T06:37:46.496 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T06:37:46.594 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398 branch: main description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} email: ceph-qa@ceph.com first_in_suite: false job_id: '7900398' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: main ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore allocator: stupid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: main sha1: 0d61375b8ab53be4c233f5b93e463edfdff9316a owner: scheduled_teuthology@teuthology priority: 950 repo: https://git.ceph.com/ceph.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 5839 sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 sleep_before_teardown: 0 subset: 99/128 suite: rbd suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 0d61375b8ab53be4c233f5b93e463edfdff9316a targets: smithi129.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCewASal6XC5348xuSGBkugXPsEDpkJbeCgS3/Cs+587P8hWBK2W/EOtSErdQ4R+RR1SNAalWl4DZy/nL/NQoSw= smithi169.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSdrVcJWaQlujfB2Sjg4WwVO1/3Ud6hUEmt60A7ZtRUWilyOrIDw3caPOYXMYTtwBVS3Qu54tSO7pau77hUBbo= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: pool RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-10_20:16:17 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-14T06:37:46.594 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa; will attempt to use it 2024-09-14T06:37:46.595 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa/tasks 2024-09-14T06:37:46.595 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T06:37:46.597 INFO:teuthology.task.internal:Checking packages... 2024-09-14T06:37:46.618 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '7f24e589328ae9e639c0009d93340e5b235d0a47' 2024-09-14T06:37:46.619 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T06:37:46.619 INFO:teuthology.packaging:ref: None 2024-09-14T06:37:46.619 INFO:teuthology.packaging:tag: None 2024-09-14T06:37:46.619 INFO:teuthology.packaging:branch: main 2024-09-14T06:37:46.619 INFO:teuthology.packaging:sha1: 7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T06:37:46.619 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=main 2024-09-14T06:37:46.851 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4907-ga2782315-1jammy 2024-09-14T06:37:46.853 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T06:37:46.867 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T06:37:46.867 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T06:37:46.967 INFO:teuthology.task.internal:Saving configuration 2024-09-14T06:37:46.986 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T06:37:46.997 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T06:37:47.017 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi129.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 06:31:55.639616', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBVXkoWlWlUsrf7hGH/vRKfLngBnZMyb5nI8xWj8gmol'} 2024-09-14T06:37:47.033 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi169.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 06:31:55.637572', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL/D+7l1yiSpJQ3w9YYTXfskkfgfP8sUJnFdLDe/UbVs'} 2024-09-14T06:37:47.033 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T06:37:47.040 INFO:teuthology.task.internal:roles: ubuntu@smithi129.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2024-09-14T06:37:47.041 INFO:teuthology.task.internal:roles: ubuntu@smithi169.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2024-09-14T06:37:47.041 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T06:37:47.128 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9d88b67010>, signals=[15]) 2024-09-14T06:37:47.128 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T06:37:47.135 INFO:teuthology.task.internal:Opening connections... 2024-09-14T06:37:47.135 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T06:37:47.136 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:37:47.214 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:37:47.215 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:37:47.291 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T06:37:47.298 DEBUG:teuthology.orchestra.run.smithi129:> uname -m 2024-09-14T06:37:47.303 INFO:teuthology.orchestra.run.smithi129.stdout:x86_64 2024-09-14T06:37:47.304 DEBUG:teuthology.orchestra.run.smithi129:> cat /etc/os-release 2024-09-14T06:37:47.351 INFO:teuthology.orchestra.run.smithi129.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:NAME="Ubuntu" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:VERSION_ID="22.04" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:VERSION_CODENAME=jammy 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:ID=ubuntu 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:ID_LIKE=debian 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-14T06:37:47.352 INFO:teuthology.orchestra.run.smithi129.stdout:UBUNTU_CODENAME=jammy 2024-09-14T06:37:47.353 INFO:teuthology.lock.ops:Updating smithi129.front.sepia.ceph.com on lock server 2024-09-14T06:37:47.373 DEBUG:teuthology.orchestra.run.smithi169:> uname -m 2024-09-14T06:37:47.378 INFO:teuthology.orchestra.run.smithi169.stdout:x86_64 2024-09-14T06:37:47.379 DEBUG:teuthology.orchestra.run.smithi169:> cat /etc/os-release 2024-09-14T06:37:47.425 INFO:teuthology.orchestra.run.smithi169.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-14T06:37:47.425 INFO:teuthology.orchestra.run.smithi169.stdout:NAME="Ubuntu" 2024-09-14T06:37:47.425 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION_ID="22.04" 2024-09-14T06:37:47.425 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-14T06:37:47.425 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION_CODENAME=jammy 2024-09-14T06:37:47.425 INFO:teuthology.orchestra.run.smithi169.stdout:ID=ubuntu 2024-09-14T06:37:47.426 INFO:teuthology.orchestra.run.smithi169.stdout:ID_LIKE=debian 2024-09-14T06:37:47.426 INFO:teuthology.orchestra.run.smithi169.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-14T06:37:47.426 INFO:teuthology.orchestra.run.smithi169.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-14T06:37:47.426 INFO:teuthology.orchestra.run.smithi169.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-14T06:37:47.426 INFO:teuthology.orchestra.run.smithi169.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-14T06:37:47.426 INFO:teuthology.orchestra.run.smithi169.stdout:UBUNTU_CODENAME=jammy 2024-09-14T06:37:47.426 INFO:teuthology.lock.ops:Updating smithi169.front.sepia.ceph.com on lock server 2024-09-14T06:37:47.448 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T06:37:47.457 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T06:37:47.481 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T06:37:47.482 DEBUG:teuthology.orchestra.run.smithi129:> test '!' -e /home/ubuntu/cephtest 2024-09-14T06:37:47.484 DEBUG:teuthology.orchestra.run.smithi169:> test '!' -e /home/ubuntu/cephtest 2024-09-14T06:37:47.488 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T06:37:47.513 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T06:37:47.513 DEBUG:teuthology.orchestra.run.smithi129:> test -z $(ls -A /var/lib/ceph) 2024-09-14T06:37:47.533 DEBUG:teuthology.orchestra.run.smithi169:> test -z $(ls -A /var/lib/ceph) 2024-09-14T06:37:47.549 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T06:37:47.601 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T06:37:47.616 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T06:37:47.616 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'kdb': 1, 'sha1': 'distro'}, 'cluster2.mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T06:37:47.616 DEBUG:teuthology.orchestra.run.smithi129:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T06:37:47.617 DEBUG:teuthology.orchestra.run.smithi169:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T06:37:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:37:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:37:47.622 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2024-09-14T06:37:47.623 DEBUG:teuthology.orchestra.run.smithi129:> uname -r 2024-09-14T06:37:47.668 INFO:teuthology.orchestra.run.smithi129.stdout:5.15.0-56-generic 2024-09-14T06:37:47.668 INFO:teuthology.task.kernel:Running kernel on smithi129: 5.15.0-56-generic 2024-09-14T06:37:47.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo apt-get clean 2024-09-14T06:37:47.669 INFO:teuthology.orchestra.run.smithi169.stdout:5.15.0-56-generic 2024-09-14T06:37:47.670 INFO:teuthology.task.kernel:Running kernel on smithi169: 5.15.0-56-generic 2024-09-14T06:37:47.670 DEBUG:teuthology.orchestra.run.smithi169:> sudo apt-get clean 2024-09-14T06:37:47.806 DEBUG:teuthology.orchestra.run.smithi129:> sudo apt-get update 2024-09-14T06:37:47.811 DEBUG:teuthology.orchestra.run.smithi169:> sudo apt-get update 2024-09-14T06:37:48.092 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-14T06:37:48.092 INFO:teuthology.orchestra.run.smithi129.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-14T06:37:48.105 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-14T06:37:48.105 INFO:teuthology.orchestra.run.smithi169.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-14T06:37:48.156 INFO:teuthology.orchestra.run.smithi129.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-14T06:37:48.157 INFO:teuthology.orchestra.run.smithi169.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-14T06:37:48.512 INFO:teuthology.orchestra.run.smithi129.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-14T06:37:48.514 INFO:teuthology.orchestra.run.smithi169.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-14T06:37:48.527 INFO:teuthology.orchestra.run.smithi129.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,806 kB] 2024-09-14T06:37:48.532 INFO:teuthology.orchestra.run.smithi169.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,806 kB] 2024-09-14T06:37:48.649 INFO:teuthology.orchestra.run.smithi129.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,023 kB] 2024-09-14T06:37:48.655 INFO:teuthology.orchestra.run.smithi169.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,023 kB] 2024-09-14T06:37:48.966 INFO:teuthology.orchestra.run.smithi129.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-14T06:37:48.966 INFO:teuthology.orchestra.run.smithi129.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [539 kB] 2024-09-14T06:37:48.973 INFO:teuthology.orchestra.run.smithi169.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-14T06:37:48.981 INFO:teuthology.orchestra.run.smithi129.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [295 kB] 2024-09-14T06:37:48.990 INFO:teuthology.orchestra.run.smithi129.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-14T06:37:48.991 INFO:teuthology.orchestra.run.smithi129.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,377 kB] 2024-09-14T06:37:48.999 INFO:teuthology.orchestra.run.smithi169.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [539 kB] 2024-09-14T06:37:49.001 INFO:teuthology.orchestra.run.smithi129.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-14T06:37:49.007 INFO:teuthology.orchestra.run.smithi169.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-14T06:37:49.015 INFO:teuthology.orchestra.run.smithi169.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [295 kB] 2024-09-14T06:37:49.022 INFO:teuthology.orchestra.run.smithi169.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-14T06:37:49.022 INFO:teuthology.orchestra.run.smithi169.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-14T06:37:49.024 INFO:teuthology.orchestra.run.smithi169.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-14T06:37:49.024 INFO:teuthology.orchestra.run.smithi169.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-14T06:37:49.025 INFO:teuthology.orchestra.run.smithi169.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,377 kB] 2024-09-14T06:37:49.025 INFO:teuthology.orchestra.run.smithi129.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-14T06:37:49.026 INFO:teuthology.orchestra.run.smithi129.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-14T06:37:49.087 INFO:teuthology.orchestra.run.smithi129.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-14T06:37:49.087 INFO:teuthology.orchestra.run.smithi129.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-14T06:37:49.089 INFO:teuthology.orchestra.run.smithi129.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [409 kB] 2024-09-14T06:37:49.089 INFO:teuthology.orchestra.run.smithi129.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-14T06:37:49.093 INFO:teuthology.orchestra.run.smithi169.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-14T06:37:49.099 INFO:teuthology.orchestra.run.smithi129.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-14T06:37:49.109 INFO:teuthology.orchestra.run.smithi169.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-14T06:37:49.110 INFO:teuthology.orchestra.run.smithi169.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-14T06:37:49.119 INFO:teuthology.orchestra.run.smithi129.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-14T06:37:49.119 INFO:teuthology.orchestra.run.smithi129.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [627 kB] 2024-09-14T06:37:49.120 INFO:teuthology.orchestra.run.smithi129.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-14T06:37:49.130 INFO:teuthology.orchestra.run.smithi129.stdout:Get:23 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [902 kB] 2024-09-14T06:37:49.133 INFO:teuthology.orchestra.run.smithi129.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-14T06:37:49.145 INFO:teuthology.orchestra.run.smithi129.stdout:Get:25 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-14T06:37:49.146 INFO:teuthology.orchestra.run.smithi169.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-14T06:37:49.146 INFO:teuthology.orchestra.run.smithi169.stdout:Get:20 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-14T06:37:49.147 INFO:teuthology.orchestra.run.smithi169.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [409 kB] 2024-09-14T06:37:49.148 INFO:teuthology.orchestra.run.smithi129.stdout:Get:26 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-14T06:37:49.151 INFO:teuthology.orchestra.run.smithi129.stdout:Get:27 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-14T06:37:49.151 INFO:teuthology.orchestra.run.smithi129.stdout:Get:28 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-14T06:37:49.158 INFO:teuthology.orchestra.run.smithi129.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-14T06:37:49.159 INFO:teuthology.orchestra.run.smithi169.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-14T06:37:49.159 INFO:teuthology.orchestra.run.smithi169.stdout:Get:23 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [902 kB] 2024-09-14T06:37:49.162 INFO:teuthology.orchestra.run.smithi129.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.1 kB] 2024-09-14T06:37:49.163 INFO:teuthology.orchestra.run.smithi129.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-14T06:37:49.163 INFO:teuthology.orchestra.run.smithi129.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-14T06:37:49.164 INFO:teuthology.orchestra.run.smithi169.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-14T06:37:49.170 INFO:teuthology.orchestra.run.smithi169.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.1 kB] 2024-09-14T06:37:49.171 INFO:teuthology.orchestra.run.smithi169.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-14T06:37:49.172 INFO:teuthology.orchestra.run.smithi169.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-14T06:37:49.173 INFO:teuthology.orchestra.run.smithi169.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-14T06:37:49.180 INFO:teuthology.orchestra.run.smithi169.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [627 kB] 2024-09-14T06:37:49.184 INFO:teuthology.orchestra.run.smithi169.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-14T06:37:49.186 INFO:teuthology.orchestra.run.smithi169.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-14T06:37:49.187 INFO:teuthology.orchestra.run.smithi169.stdout:Get:32 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-14T06:37:49.187 INFO:teuthology.orchestra.run.smithi169.stdout:Get:33 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-14T06:37:49.212 INFO:teuthology.orchestra.run.smithi129.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-14T06:37:49.212 INFO:teuthology.orchestra.run.smithi129.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-14T06:37:49.213 INFO:teuthology.orchestra.run.smithi129.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-14T06:37:49.214 INFO:teuthology.orchestra.run.smithi129.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-14T06:37:49.219 INFO:teuthology.orchestra.run.smithi169.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-14T06:37:49.219 INFO:teuthology.orchestra.run.smithi169.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-14T06:37:49.220 INFO:teuthology.orchestra.run.smithi169.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-14T06:37:49.227 INFO:teuthology.orchestra.run.smithi129.stdout:Get:37 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-14T06:37:49.228 INFO:teuthology.orchestra.run.smithi129.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-14T06:37:49.248 INFO:teuthology.orchestra.run.smithi129.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-14T06:37:49.248 INFO:teuthology.orchestra.run.smithi129.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-14T06:37:49.249 INFO:teuthology.orchestra.run.smithi129.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-14T06:37:49.249 INFO:teuthology.orchestra.run.smithi129.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-14T06:37:49.250 INFO:teuthology.orchestra.run.smithi129.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-14T06:37:49.255 INFO:teuthology.orchestra.run.smithi169.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-14T06:37:49.256 INFO:teuthology.orchestra.run.smithi169.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-14T06:37:49.256 INFO:teuthology.orchestra.run.smithi169.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-14T06:37:49.257 INFO:teuthology.orchestra.run.smithi169.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-14T06:37:49.257 INFO:teuthology.orchestra.run.smithi169.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-14T06:37:49.265 INFO:teuthology.orchestra.run.smithi169.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-14T06:37:49.265 INFO:teuthology.orchestra.run.smithi169.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-14T06:37:49.293 INFO:teuthology.orchestra.run.smithi129.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-14T06:37:49.300 INFO:teuthology.orchestra.run.smithi169.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-14T06:37:55.165 INFO:teuthology.orchestra.run.smithi129.stdout:Fetched 16.0 MB in 3s (5,749 kB/s) 2024-09-14T06:37:55.494 INFO:teuthology.orchestra.run.smithi169.stdout:Fetched 16.0 MB in 3s (5,679 kB/s) 2024-09-14T06:37:56.635 INFO:teuthology.orchestra.run.smithi129.stdout:Reading package lists... 2024-09-14T06:37:56.655 DEBUG:teuthology.orchestra.run.smithi129:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-14T06:37:56.712 INFO:teuthology.orchestra.run.smithi129.stdout:Reading package lists... 2024-09-14T06:37:56.906 INFO:teuthology.orchestra.run.smithi129.stdout:Building dependency tree... 2024-09-14T06:37:56.907 INFO:teuthology.orchestra.run.smithi129.stdout:Reading state information... 2024-09-14T06:37:56.915 INFO:teuthology.orchestra.run.smithi169.stdout:Reading package lists... 2024-09-14T06:37:56.933 DEBUG:teuthology.orchestra.run.smithi169:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-14T06:37:56.989 INFO:teuthology.orchestra.run.smithi169.stdout:Reading package lists... 2024-09-14T06:37:57.071 INFO:teuthology.orchestra.run.smithi129.stdout:The following packages were automatically installed and are no longer required: 2024-09-14T06:37:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-14T06:37:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-14T06:37:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-14T06:37:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-14T06:37:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: python2.7-minimal 2024-09-14T06:37:57.072 INFO:teuthology.orchestra.run.smithi129.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-14T06:37:57.073 INFO:teuthology.orchestra.run.smithi129.stdout:The following additional packages will be installed: 2024-09-14T06:37:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-14T06:37:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-14T06:37:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-14T06:37:57.074 INFO:teuthology.orchestra.run.smithi129.stdout:Suggested packages: 2024-09-14T06:37:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-14T06:37:57.074 INFO:teuthology.orchestra.run.smithi129.stdout:Recommended packages: 2024-09-14T06:37:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: thermald 2024-09-14T06:37:57.117 INFO:teuthology.orchestra.run.smithi129.stdout:The following NEW packages will be installed: 2024-09-14T06:37:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-14T06:37:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-14T06:37:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-14T06:37:57.119 INFO:teuthology.orchestra.run.smithi129.stdout:The following packages will be upgraded: 2024-09-14T06:37:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-14T06:37:57.171 INFO:teuthology.orchestra.run.smithi129.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-14T06:37:57.172 INFO:teuthology.orchestra.run.smithi129.stdout:Need to get 113 MB of archives. 2024-09-14T06:37:57.172 INFO:teuthology.orchestra.run.smithi129.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-14T06:37:57.172 INFO:teuthology.orchestra.run.smithi129.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-14T06:37:57.176 INFO:teuthology.orchestra.run.smithi169.stdout:Building dependency tree... 2024-09-14T06:37:57.176 INFO:teuthology.orchestra.run.smithi169.stdout:Reading state information... 2024-09-14T06:37:57.342 INFO:teuthology.orchestra.run.smithi169.stdout:The following packages were automatically installed and are no longer required: 2024-09-14T06:37:57.342 INFO:teuthology.orchestra.run.smithi169.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-14T06:37:57.342 INFO:teuthology.orchestra.run.smithi169.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-14T06:37:57.343 INFO:teuthology.orchestra.run.smithi169.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-14T06:37:57.343 INFO:teuthology.orchestra.run.smithi169.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-14T06:37:57.343 INFO:teuthology.orchestra.run.smithi169.stdout: python2.7-minimal 2024-09-14T06:37:57.343 INFO:teuthology.orchestra.run.smithi169.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-14T06:37:57.344 INFO:teuthology.orchestra.run.smithi169.stdout:The following additional packages will be installed: 2024-09-14T06:37:57.345 INFO:teuthology.orchestra.run.smithi169.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-14T06:37:57.345 INFO:teuthology.orchestra.run.smithi169.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-14T06:37:57.346 INFO:teuthology.orchestra.run.smithi169.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-14T06:37:57.346 INFO:teuthology.orchestra.run.smithi169.stdout:Suggested packages: 2024-09-14T06:37:57.346 INFO:teuthology.orchestra.run.smithi169.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-14T06:37:57.346 INFO:teuthology.orchestra.run.smithi169.stdout:Recommended packages: 2024-09-14T06:37:57.347 INFO:teuthology.orchestra.run.smithi169.stdout: thermald 2024-09-14T06:37:57.396 INFO:teuthology.orchestra.run.smithi169.stdout:The following NEW packages will be installed: 2024-09-14T06:37:57.396 INFO:teuthology.orchestra.run.smithi169.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-14T06:37:57.396 INFO:teuthology.orchestra.run.smithi169.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-14T06:37:57.396 INFO:teuthology.orchestra.run.smithi169.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-14T06:37:57.398 INFO:teuthology.orchestra.run.smithi169.stdout:The following packages will be upgraded: 2024-09-14T06:37:57.398 INFO:teuthology.orchestra.run.smithi169.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-14T06:37:57.578 INFO:teuthology.orchestra.run.smithi169.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-14T06:37:57.578 INFO:teuthology.orchestra.run.smithi169.stdout:Need to get 113 MB of archives. 2024-09-14T06:37:57.578 INFO:teuthology.orchestra.run.smithi169.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-14T06:37:57.578 INFO:teuthology.orchestra.run.smithi169.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-14T06:37:57.603 INFO:teuthology.orchestra.run.smithi129.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-14T06:37:57.703 INFO:teuthology.orchestra.run.smithi129.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-14T06:37:58.527 INFO:teuthology.orchestra.run.smithi129.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-14T06:37:58.527 INFO:teuthology.orchestra.run.smithi129.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-14T06:37:58.527 INFO:teuthology.orchestra.run.smithi129.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-14T06:37:58.649 INFO:teuthology.orchestra.run.smithi129.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-14T06:37:58.671 INFO:teuthology.orchestra.run.smithi129.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-14T06:37:59.065 INFO:teuthology.orchestra.run.smithi129.stdout:Fetched 113 MB in 2s (73.3 MB/s) 2024-09-14T06:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-14T06:37:59.276 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-14T06:37:59.427 INFO:teuthology.orchestra.run.smithi169.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-14T06:38:01.236 INFO:teuthology.orchestra.run.smithi129.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-14T06:38:01.241 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:01.415 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:01.657 INFO:teuthology.orchestra.run.smithi169.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-14T06:38:01.657 INFO:teuthology.orchestra.run.smithi169.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-14T06:38:01.658 INFO:teuthology.orchestra.run.smithi169.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-14T06:38:02.022 INFO:teuthology.orchestra.run.smithi169.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-14T06:38:02.103 INFO:teuthology.orchestra.run.smithi169.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-14T06:38:02.503 INFO:teuthology.orchestra.run.smithi169.stdout:Fetched 113 MB in 5s (24.1 MB/s) 2024-09-14T06:38:02.678 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-14T06:38:03.540 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-14T06:38:03.562 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:03.642 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:04.043 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-14T06:38:04.064 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:04.093 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:04.675 INFO:teuthology.orchestra.run.smithi169.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-14T06:38:04.679 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:04.803 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:07.162 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-14T06:38:07.184 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:07.270 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:07.732 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-14T06:38:07.752 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:07.799 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:10.949 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-14T06:38:11.068 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-14T06:38:11.401 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-14T06:38:11.538 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-14T06:38:11.832 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-14T06:38:11.843 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-14T06:38:11.882 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-14T06:38:14.214 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-14T06:38:14.350 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-14T06:38:14.682 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-14T06:38:14.803 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-14T06:38:15.098 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-14T06:38:15.121 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-14T06:38:15.156 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-14T06:38:17.273 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-14T06:38:17.310 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:17.348 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:20.068 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-14T06:38:20.192 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-14T06:38:20.517 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-14T06:38:20.638 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:20.790 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-14T06:38:20.857 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-14T06:38:20.894 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-14T06:38:20.917 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:20.931 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:22.237 INFO:teuthology.orchestra.run.smithi129.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-14T06:38:22.238 INFO:teuthology.orchestra.run.smithi129.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-14T06:38:22.238 INFO:teuthology.orchestra.run.smithi129.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-14T06:38:22.238 INFO:teuthology.orchestra.run.smithi129.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:22.404 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:24.469 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-14T06:38:24.555 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:24.595 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-14T06:38:24.961 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-14T06:38:25.067 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:25.176 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-14T06:38:25.304 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:25.957 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-14T06:38:26.082 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-14T06:38:26.200 INFO:teuthology.orchestra.run.smithi129.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:26.261 INFO:teuthology.orchestra.run.smithi129.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-14T06:38:26.261 INFO:teuthology.orchestra.run.smithi129.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:26.639 INFO:teuthology.orchestra.run.smithi169.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-14T06:38:26.639 INFO:teuthology.orchestra.run.smithi169.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-14T06:38:26.639 INFO:teuthology.orchestra.run.smithi169.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-14T06:38:26.639 INFO:teuthology.orchestra.run.smithi169.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:26.799 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:28.200 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:30.144 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-14T06:38:30.578 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-14T06:38:30.704 INFO:teuthology.orchestra.run.smithi169.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-14T06:38:30.768 INFO:teuthology.orchestra.run.smithi169.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-14T06:38:30.769 INFO:teuthology.orchestra.run.smithi169.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:40.837 INFO:teuthology.orchestra.run.smithi129.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-14T06:38:40.837 INFO:teuthology.orchestra.run.smithi129.stdout:Sourcing file `/etc/default/grub' 2024-09-14T06:38:40.847 INFO:teuthology.orchestra.run.smithi129.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-14T06:38:40.881 INFO:teuthology.orchestra.run.smithi129.stdout:Generating grub configuration file ... 2024-09-14T06:38:41.166 INFO:teuthology.orchestra.run.smithi129.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-14T06:38:41.181 INFO:teuthology.orchestra.run.smithi129.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:41.466 INFO:teuthology.orchestra.run.smithi129.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-14T06:38:41.471 INFO:teuthology.orchestra.run.smithi129.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-14T06:38:41.542 INFO:teuthology.orchestra.run.smithi129.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-14T06:38:41.548 INFO:teuthology.orchestra.run.smithi129.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-14T06:38:41.711 INFO:teuthology.orchestra.run.smithi129.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-14T06:38:41.711 INFO:teuthology.orchestra.run.smithi129.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-14T06:38:41.711 INFO:teuthology.orchestra.run.smithi129.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-14T06:38:41.758 INFO:teuthology.orchestra.run.smithi129.stdout:done 2024-09-14T06:38:42.212 DEBUG:teuthology.orchestra.run.smithi129:> dpkg -s linux-image-generic 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Package: linux-image-generic 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Status: install ok installed 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Priority: optional 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Section: kernel 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Installed-Size: 21 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Maintainer: Ubuntu Kernel Team 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Architecture: amd64 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Source: linux-meta 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Version: 5.15.0.121.121 2024-09-14T06:38:42.238 INFO:teuthology.orchestra.run.smithi129.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-14T06:38:42.239 INFO:teuthology.orchestra.run.smithi129.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-14T06:38:42.239 INFO:teuthology.orchestra.run.smithi129.stdout:Recommends: thermald 2024-09-14T06:38:42.239 INFO:teuthology.orchestra.run.smithi129.stdout:Description: Generic Linux kernel image 2024-09-14T06:38:42.239 INFO:teuthology.orchestra.run.smithi129.stdout: This package will always depend on the latest generic kernel image 2024-09-14T06:38:42.239 INFO:teuthology.orchestra.run.smithi129.stdout: available. 2024-09-14T06:38:42.239 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-14T06:38:42.239 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T06:38:42.239 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T06:38:42.239 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a... 2024-09-14T06:38:42.240 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi129.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T06:38:42.240 DEBUG:teuthology.orchestra.run.smithi129:> sudo apt-get clean 2024-09-14T06:38:42.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo apt-get update 2024-09-14T06:38:42.641 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-14T06:38:42.641 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-14T06:38:42.713 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-14T06:38:42.795 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-14T06:38:44.055 INFO:teuthology.orchestra.run.smithi129.stdout:Reading package lists... 2024-09-14T06:38:44.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-14T06:38:44.134 INFO:teuthology.orchestra.run.smithi129.stdout:Reading package lists... 2024-09-14T06:38:44.334 INFO:teuthology.orchestra.run.smithi129.stdout:Building dependency tree... 2024-09-14T06:38:44.335 INFO:teuthology.orchestra.run.smithi129.stdout:Reading state information... 2024-09-14T06:38:44.502 INFO:teuthology.orchestra.run.smithi129.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-14T06:38:44.502 INFO:teuthology.orchestra.run.smithi129.stdout:The following packages were automatically installed and are no longer required: 2024-09-14T06:38:44.502 INFO:teuthology.orchestra.run.smithi129.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-14T06:38:44.502 INFO:teuthology.orchestra.run.smithi129.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-14T06:38:44.503 INFO:teuthology.orchestra.run.smithi129.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-14T06:38:44.503 INFO:teuthology.orchestra.run.smithi129.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-14T06:38:44.503 INFO:teuthology.orchestra.run.smithi129.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-14T06:38:44.503 INFO:teuthology.orchestra.run.smithi129.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-14T06:38:44.543 INFO:teuthology.orchestra.run.smithi129.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-14T06:38:44.545 DEBUG:teuthology.orchestra.run.smithi129:> dpkg -s linux-image-generic 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Package: linux-image-generic 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Status: install ok installed 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Priority: optional 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Section: kernel 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Installed-Size: 21 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Maintainer: Ubuntu Kernel Team 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Architecture: amd64 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Source: linux-meta 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Version: 5.15.0.121.121 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-14T06:38:44.561 INFO:teuthology.orchestra.run.smithi129.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-14T06:38:44.562 INFO:teuthology.orchestra.run.smithi129.stdout:Recommends: thermald 2024-09-14T06:38:44.562 INFO:teuthology.orchestra.run.smithi129.stdout:Description: Generic Linux kernel image 2024-09-14T06:38:44.562 INFO:teuthology.orchestra.run.smithi129.stdout: This package will always depend on the latest generic kernel image 2024-09-14T06:38:44.562 INFO:teuthology.orchestra.run.smithi129.stdout: available. 2024-09-14T06:38:44.562 DEBUG:teuthology.orchestra.run.smithi129:> mktemp 2024-09-14T06:38:44.606 INFO:teuthology.orchestra.run.smithi129.stdout:/tmp/tmp.UOFqALJEWH 2024-09-14T06:38:44.606 DEBUG:teuthology.orchestra.run.smithi129:> sudo cp /boot/grub/grub.cfg /tmp/tmp.UOFqALJEWH 2024-09-14T06:38:44.640 INFO:teuthology.orchestra.run.smithi169.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-14T06:38:44.640 INFO:teuthology.orchestra.run.smithi169.stdout:Sourcing file `/etc/default/grub' 2024-09-14T06:38:44.650 INFO:teuthology.orchestra.run.smithi169.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-14T06:38:44.661 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0666 /tmp/tmp.UOFqALJEWH 2024-09-14T06:38:44.666 INFO:teuthology.orchestra.run.smithi169.stdout:Generating grub configuration file ... 2024-09-14T06:38:44.774 DEBUG:teuthology.orchestra.remote:smithi129:/tmp/tmp.UOFqALJEWH is 10KB 2024-09-14T06:38:44.786 DEBUG:teuthology.orchestra.run.smithi129:> rm -fr /tmp/tmp.UOFqALJEWH 2024-09-14T06:38:44.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-14T06:38:44.844 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:38:44.845 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-14T06:38:44.845 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-14T06:38:44.907 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-14T06:38:44.908 DEBUG:teuthology.orchestra.run.smithi129:> sudo update-grub 2024-09-14T06:38:44.952 INFO:teuthology.orchestra.run.smithi169.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-14T06:38:44.973 INFO:teuthology.orchestra.run.smithi169.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:45.254 INFO:teuthology.orchestra.run.smithi169.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-14T06:38:45.259 INFO:teuthology.orchestra.run.smithi169.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-14T06:38:45.323 INFO:teuthology.orchestra.run.smithi169.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-14T06:38:45.329 INFO:teuthology.orchestra.run.smithi169.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-14T06:38:45.494 INFO:teuthology.orchestra.run.smithi169.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-14T06:38:45.494 INFO:teuthology.orchestra.run.smithi169.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-14T06:38:45.494 INFO:teuthology.orchestra.run.smithi169.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-14T06:38:45.534 INFO:teuthology.orchestra.run.smithi169.stdout:done 2024-09-14T06:38:46.025 DEBUG:teuthology.orchestra.run.smithi169:> dpkg -s linux-image-generic 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Package: linux-image-generic 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Status: install ok installed 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Priority: optional 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Section: kernel 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Installed-Size: 21 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Maintainer: Ubuntu Kernel Team 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Architecture: amd64 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Source: linux-meta 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Version: 5.15.0.121.121 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-14T06:38:46.054 INFO:teuthology.orchestra.run.smithi169.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-14T06:38:46.055 INFO:teuthology.orchestra.run.smithi169.stdout:Recommends: thermald 2024-09-14T06:38:46.055 INFO:teuthology.orchestra.run.smithi169.stdout:Description: Generic Linux kernel image 2024-09-14T06:38:46.055 INFO:teuthology.orchestra.run.smithi169.stdout: This package will always depend on the latest generic kernel image 2024-09-14T06:38:46.055 INFO:teuthology.orchestra.run.smithi169.stdout: available. 2024-09-14T06:38:46.055 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-14T06:38:46.055 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T06:38:46.055 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T06:38:46.055 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a... 2024-09-14T06:38:46.055 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi169.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T06:38:46.056 DEBUG:teuthology.orchestra.run.smithi169:> sudo apt-get clean 2024-09-14T06:38:46.179 INFO:teuthology.orchestra.run.smithi129.stderr:Sourcing file `/etc/default/grub' 2024-09-14T06:38:46.180 INFO:teuthology.orchestra.run.smithi129.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-14T06:38:46.181 INFO:teuthology.orchestra.run.smithi129.stderr:Generating grub configuration file ... 2024-09-14T06:38:46.242 DEBUG:teuthology.orchestra.run.smithi169:> sudo apt-get update 2024-09-14T06:38:46.368 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-14T06:38:46.372 INFO:teuthology.orchestra.run.smithi129.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-14T06:38:46.378 INFO:teuthology.orchestra.run.smithi129.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:46.491 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-14T06:38:46.571 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-14T06:38:46.615 INFO:teuthology.orchestra.run.smithi129.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-14T06:38:46.620 INFO:teuthology.orchestra.run.smithi129.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-14T06:38:46.653 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-14T06:38:46.660 INFO:teuthology.orchestra.run.smithi129.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-14T06:38:46.665 INFO:teuthology.orchestra.run.smithi129.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-14T06:38:46.787 INFO:teuthology.orchestra.run.smithi129.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-14T06:38:46.787 INFO:teuthology.orchestra.run.smithi129.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-14T06:38:46.787 INFO:teuthology.orchestra.run.smithi129.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-14T06:38:46.802 INFO:teuthology.orchestra.run.smithi129.stderr:done 2024-09-14T06:38:46.804 DEBUG:teuthology.orchestra.run.smithi129:> sudo shutdown -r now 2024-09-14T06:38:47.841 INFO:teuthology.orchestra.run.smithi169.stdout:Reading package lists... 2024-09-14T06:38:47.860 DEBUG:teuthology.orchestra.run.smithi169:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-14T06:38:47.923 INFO:teuthology.orchestra.run.smithi169.stdout:Reading package lists... 2024-09-14T06:38:48.133 INFO:teuthology.orchestra.run.smithi169.stdout:Building dependency tree... 2024-09-14T06:38:48.134 INFO:teuthology.orchestra.run.smithi169.stdout:Reading state information... 2024-09-14T06:38:48.313 INFO:teuthology.orchestra.run.smithi169.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout:The following packages were automatically installed and are no longer required: 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-14T06:38:48.314 INFO:teuthology.orchestra.run.smithi169.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-14T06:38:48.355 INFO:teuthology.orchestra.run.smithi169.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-14T06:38:48.357 DEBUG:teuthology.orchestra.run.smithi169:> dpkg -s linux-image-generic 2024-09-14T06:38:48.372 INFO:teuthology.orchestra.run.smithi169.stdout:Package: linux-image-generic 2024-09-14T06:38:48.372 INFO:teuthology.orchestra.run.smithi169.stdout:Status: install ok installed 2024-09-14T06:38:48.372 INFO:teuthology.orchestra.run.smithi169.stdout:Priority: optional 2024-09-14T06:38:48.372 INFO:teuthology.orchestra.run.smithi169.stdout:Section: kernel 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Installed-Size: 21 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Maintainer: Ubuntu Kernel Team 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Architecture: amd64 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Source: linux-meta 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Version: 5.15.0.121.121 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Recommends: thermald 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout:Description: Generic Linux kernel image 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout: This package will always depend on the latest generic kernel image 2024-09-14T06:38:48.373 INFO:teuthology.orchestra.run.smithi169.stdout: available. 2024-09-14T06:38:48.374 DEBUG:teuthology.orchestra.run.smithi169:> mktemp 2024-09-14T06:38:48.422 INFO:teuthology.orchestra.run.smithi169.stdout:/tmp/tmp.kOzBnMnRel 2024-09-14T06:38:48.422 DEBUG:teuthology.orchestra.run.smithi169:> sudo cp /boot/grub/grub.cfg /tmp/tmp.kOzBnMnRel 2024-09-14T06:38:48.474 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0666 /tmp/tmp.kOzBnMnRel 2024-09-14T06:38:48.582 DEBUG:teuthology.orchestra.remote:smithi169:/tmp/tmp.kOzBnMnRel is 10KB 2024-09-14T06:38:48.595 DEBUG:teuthology.orchestra.run.smithi169:> rm -fr /tmp/tmp.kOzBnMnRel 2024-09-14T06:38:48.601 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-14T06:38:48.656 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:38:48.656 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-14T06:38:48.656 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-14T06:38:48.723 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-14T06:38:48.723 DEBUG:teuthology.orchestra.run.smithi169:> sudo update-grub 2024-09-14T06:38:49.883 INFO:teuthology.orchestra.run.smithi169.stderr:Sourcing file `/etc/default/grub' 2024-09-14T06:38:49.884 INFO:teuthology.orchestra.run.smithi169.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-14T06:38:49.886 INFO:teuthology.orchestra.run.smithi169.stderr:Generating grub configuration file ... 2024-09-14T06:38:50.067 INFO:teuthology.orchestra.run.smithi169.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-14T06:38:50.072 INFO:teuthology.orchestra.run.smithi169.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-14T06:38:50.299 INFO:teuthology.orchestra.run.smithi169.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-14T06:38:50.305 INFO:teuthology.orchestra.run.smithi169.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-14T06:38:50.348 INFO:teuthology.orchestra.run.smithi169.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-14T06:38:50.354 INFO:teuthology.orchestra.run.smithi169.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-14T06:38:50.486 INFO:teuthology.orchestra.run.smithi169.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-14T06:38:50.486 INFO:teuthology.orchestra.run.smithi169.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-14T06:38:50.486 INFO:teuthology.orchestra.run.smithi169.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-14T06:38:50.505 INFO:teuthology.orchestra.run.smithi169.stderr:done 2024-09-14T06:38:50.514 DEBUG:teuthology.orchestra.run.smithi169:> sudo shutdown -r now 2024-09-14T06:39:16.833 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version... 2024-09-14T06:39:16.835 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:39:16.837 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:39:20.521 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version... 2024-09-14T06:39:20.521 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:39:20.522 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:39:35.229 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.129 2024-09-14T06:39:39.069 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2024-09-14T06:39:44.233 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:39:44.234 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:39:47.293 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.129 2024-09-14T06:39:48.070 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:39:48.071 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:39:51.133 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2024-09-14T06:39:59.301 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:39:59.302 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:40:03.137 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:40:03.137 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:40:17.729 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.129 2024-09-14T06:40:32.744 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:40:32.745 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:40:33.076 DEBUG:teuthology.orchestra.run.smithi129:> true 2024-09-14T06:40:33.696 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:40:33.696 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.15.0-121-generic"... 2024-09-14T06:40:33.697 DEBUG:teuthology.orchestra.run.smithi129:> uname -r 2024-09-14T06:40:33.745 INFO:teuthology.orchestra.run.smithi129.stdout:5.15.0-121-generic 2024-09-14T06:40:33.745 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-14T06:40:33.745 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T06:40:33.745 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-14T06:40:34.746 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2024-09-14T06:40:34.746 DEBUG:teuthology.orchestra.run.smithi129:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T06:40:34.847 INFO:teuthology.orchestra.run.smithi129.stdout:ttyS1 2024-09-14T06:40:34.866 DEBUG:teuthology.parallel:result is None 2024-09-14T06:41:03.140 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T06:41:18.141 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:41:18.143 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:41:18.458 DEBUG:teuthology.orchestra.run.smithi169:> true 2024-09-14T06:41:19.043 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:41:19.043 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.15.0-121-generic"... 2024-09-14T06:41:19.043 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2024-09-14T06:41:19.090 INFO:teuthology.orchestra.run.smithi169.stdout:5.15.0-121-generic 2024-09-14T06:41:19.091 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-14T06:41:19.091 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T06:41:19.091 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-14T06:41:20.091 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2024-09-14T06:41:20.092 DEBUG:teuthology.orchestra.run.smithi169:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T06:41:20.161 INFO:teuthology.orchestra.run.smithi169.stdout:ttyS1 2024-09-14T06:41:20.188 DEBUG:teuthology.parallel:result is None 2024-09-14T06:41:20.188 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T06:41:20.199 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T06:41:20.199 DEBUG:teuthology.orchestra.run.smithi129:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T06:41:20.202 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T06:41:20.208 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T06:41:20.238 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T06:41:20.271 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T06:41:20.271 DEBUG:teuthology.orchestra.run.smithi129:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T06:41:20.274 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T06:41:20.294 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T06:41:20.304 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T06:41:20.304 DEBUG:teuthology.orchestra.run.smithi129:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T06:41:20.333 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T06:41:20.351 INFO:teuthology.orchestra.run.smithi129.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T06:41:20.356 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T06:41:20.359 INFO:teuthology.orchestra.run.smithi129.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T06:41:20.364 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T06:41:20.366 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T06:41:20.372 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T06:41:20.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T06:41:20.405 DEBUG:teuthology.orchestra.run.smithi169:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T06:41:20.423 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T06:41:20.432 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T06:41:20.432 DEBUG:teuthology.orchestra.run.smithi129:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T06:41:20.469 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T06:41:20.475 DEBUG:teuthology.orchestra.run.smithi129:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T06:41:20.520 DEBUG:teuthology.orchestra.run.smithi129:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T06:41:20.568 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:41:20.568 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T06:41:20.629 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T06:41:20.635 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T06:41:20.683 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:41:20.683 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T06:41:20.754 DEBUG:teuthology.orchestra.run.smithi129:> sudo service rsyslog restart 2024-09-14T06:41:20.756 DEBUG:teuthology.orchestra.run.smithi169:> sudo service rsyslog restart 2024-09-14T06:41:20.833 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T06:41:20.840 INFO:teuthology.task.internal:Starting timer... 2024-09-14T06:41:20.841 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T06:41:20.891 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T06:41:20.924 DEBUG:teuthology.task.selinux:Excluding smithi129: OS 'ubuntu' does not support SELinux 2024-09-14T06:41:20.924 DEBUG:teuthology.task.selinux:Excluding smithi169: OS 'ubuntu' does not support SELinux 2024-09-14T06:41:20.924 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T06:41:20.925 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-09-14T06:41:20.925 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T06:41:20.925 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T06:41:20.957 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-14T06:41:20.960 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-14T06:41:21.075 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T06:41:21.111 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T06:41:21.112 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi129.front.sepia.ceph.com,smithi169.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T06:48:49.310 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi129.front.sepia.ceph.com'), Remote(name='ubuntu@smithi169.front.sepia.ceph.com')] 2024-09-14T06:48:49.311 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:48:49.313 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:48:49.393 DEBUG:teuthology.orchestra.run.smithi129:> true 2024-09-14T06:48:49.472 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T06:48:49.473 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:48:49.473 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T06:48:49.552 DEBUG:teuthology.orchestra.run.smithi169:> true 2024-09-14T06:48:49.628 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2024-09-14T06:48:49.629 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T06:48:49.640 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T06:48:49.640 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T06:48:49.641 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T06:48:49.643 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T06:48:49.643 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Command line: ntpd -gq 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: ---------------------------------------------------- 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: ntp-4 is maintained by Network Time Foundation, 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: corporation. Support and training for ntp-4 are 2024-09-14T06:48:49.667 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: available at https://www.nwtime.org/support 2024-09-14T06:48:49.668 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: ---------------------------------------------------- 2024-09-14T06:48:49.668 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: proto: precision = 0.045 usec (-24) 2024-09-14T06:48:49.668 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: basedate set to 2022-02-04 2024-09-14T06:48:49.668 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: gps base set to 2022-02-06 (week 2196) 2024-09-14T06:48:49.669 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-14T06:48:49.669 INFO:teuthology.orchestra.run.smithi129.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-14T06:48:49.669 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: restrict ::: KOD does nothing without LIMITED. 2024-09-14T06:48:49.669 INFO:teuthology.orchestra.run.smithi129.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-14T06:48:49.670 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listen and drop on 0 v6wildcard [::]:123 2024-09-14T06:48:49.670 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-14T06:48:49.671 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listen normally on 2 lo 127.0.0.1:123 2024-09-14T06:48:49.671 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listen normally on 3 enp3s0f1 172.21.15.129:123 2024-09-14T06:48:49.671 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listen normally on 4 lo [::1]:123 2024-09-14T06:48:49.671 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7261%5]:123 2024-09-14T06:48:49.671 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:49 ntpd[18469]: Listening on routing socket on fd #22 for interface updates 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Command line: ntpd -gq 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: ---------------------------------------------------- 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: ntp-4 is maintained by Network Time Foundation, 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: corporation. Support and training for ntp-4 are 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: available at https://www.nwtime.org/support 2024-09-14T06:48:49.697 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: ---------------------------------------------------- 2024-09-14T06:48:49.698 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: proto: precision = 0.062 usec (-24) 2024-09-14T06:48:49.698 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: basedate set to 2022-02-04 2024-09-14T06:48:49.698 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: gps base set to 2022-02-06 (week 2196) 2024-09-14T06:48:49.698 INFO:teuthology.orchestra.run.smithi169.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-14T06:48:49.699 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-14T06:48:49.699 INFO:teuthology.orchestra.run.smithi169.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-14T06:48:49.700 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: restrict ::: KOD does nothing without LIMITED. 2024-09-14T06:48:49.700 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listen and drop on 0 v6wildcard [::]:123 2024-09-14T06:48:49.700 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-14T06:48:49.700 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listen normally on 2 lo 127.0.0.1:123 2024-09-14T06:48:49.700 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listen normally on 3 enp3s0f1 172.21.15.169:123 2024-09-14T06:48:49.700 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listen normally on 4 lo [::1]:123 2024-09-14T06:48:49.701 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7f4d%5]:123 2024-09-14T06:48:49.701 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:49 ntpd[18464]: Listening on routing socket on fd #22 for interface updates 2024-09-14T06:48:50.668 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.668 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.668 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:50.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:50 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:50.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:50.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:50 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:52.668 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.668 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.668 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:52.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:52 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:52.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:52.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:52.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:52.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:52.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:52 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:54.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:54.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:54 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:54.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:54.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:54.700 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:54.700 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:54.700 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:54 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:56.669 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.670 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stdout:14 Sep 06:48:56 ntpd[18469]: ntpd: time slew +0.000472 s 2024-09-14T06:48:56.671 INFO:teuthology.orchestra.run.smithi129.stdout:ntpd: time slew +0.000472s 2024-09-14T06:48:56.672 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-14T06:48:56.672 INFO:teuthology.orchestra.run.smithi129.stderr:14 Sep 06:48:56 ntpd[18469]: can't open /var/log/ntpstats/loopstats.20240914: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: can't open /var/log/ntpstats/rawstats.20240914: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-14T06:48:56.698 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: can't open /var/log/ntpstats/peerstats.20240914: Permission denied 2024-09-14T06:48:56.699 INFO:teuthology.orchestra.run.smithi169.stdout:14 Sep 06:48:56 ntpd[18464]: ntpd: time slew -0.000156 s 2024-09-14T06:48:56.699 INFO:teuthology.orchestra.run.smithi169.stdout:ntpd: time slew -0.000156s 2024-09-14T06:48:56.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-14T06:48:56.699 INFO:teuthology.orchestra.run.smithi169.stderr:14 Sep 06:48:56 ntpd[18464]: can't open /var/log/ntpstats/loopstats.20240914: Permission denied 2024-09-14T06:48:56.723 INFO:teuthology.orchestra.run.smithi129.stdout: remote refid st t when poll reach delay offset jitter 2024-09-14T06:48:56.723 INFO:teuthology.orchestra.run.smithi129.stdout:============================================================================== 2024-09-14T06:48:56.723 INFO:teuthology.orchestra.run.smithi129.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.723 INFO:teuthology.orchestra.run.smithi129.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.723 INFO:teuthology.orchestra.run.smithi129.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.723 INFO:teuthology.orchestra.run.smithi129.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.745 INFO:teuthology.orchestra.run.smithi169.stdout: remote refid st t when poll reach delay offset jitter 2024-09-14T06:48:56.745 INFO:teuthology.orchestra.run.smithi169.stdout:============================================================================== 2024-09-14T06:48:56.745 INFO:teuthology.orchestra.run.smithi169.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.745 INFO:teuthology.orchestra.run.smithi169.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.745 INFO:teuthology.orchestra.run.smithi169.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.746 INFO:teuthology.orchestra.run.smithi169.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-14T06:48:56.746 INFO:teuthology.run_tasks:Running task install... 2024-09-14T06:48:56.755 DEBUG:teuthology.task.install:project ceph 2024-09-14T06:48:56.755 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}} 2024-09-14T06:48:56.755 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'} 2024-09-14T06:48:56.755 INFO:teuthology.task.install:Using flavor: default 2024-09-14T06:48:56.761 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T06:48:56.761 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2024-09-14T06:48:56.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo apt-key list | grep Ceph 2024-09-14T06:48:56.762 DEBUG:teuthology.orchestra.run.smithi169:> sudo apt-key list | grep Ceph 2024-09-14T06:48:56.818 INFO:teuthology.orchestra.run.smithi129.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-14T06:48:56.845 INFO:teuthology.orchestra.run.smithi129.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-14T06:48:56.845 INFO:teuthology.orchestra.run.smithi129.stdout:uid [ unknown] Ceph.com (release key) 2024-09-14T06:48:56.845 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2024-09-14T06:48:56.846 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T06:48:56.850 INFO:teuthology.orchestra.run.smithi169.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-14T06:48:56.880 INFO:teuthology.orchestra.run.smithi169.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-14T06:48:56.880 INFO:teuthology.orchestra.run.smithi169.stdout:uid [ unknown] Ceph.com (release key) 2024-09-14T06:48:56.880 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2024-09-14T06:48:56.880 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T06:48:56.994 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default/ 2024-09-14T06:48:56.995 INFO:teuthology.task.install.deb:Package version is 19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:48:57.017 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default/ 2024-09-14T06:48:57.017 INFO:teuthology.task.install.deb:Package version is 19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:48:57.109 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:48:57.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-14T06:48:57.123 DEBUG:teuthology.orchestra.run.smithi129:> sudo apt-get update 2024-09-14T06:48:57.128 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:48:57.128 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-14T06:48:57.142 DEBUG:teuthology.orchestra.run.smithi169:> sudo apt-get update 2024-09-14T06:48:57.309 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-14T06:48:57.318 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-14T06:48:57.335 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-14T06:48:57.345 INFO:teuthology.orchestra.run.smithi129.stdout:Ign:1 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy InRelease 2024-09-14T06:48:57.362 INFO:teuthology.orchestra.run.smithi169.stdout:Ign:4 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy InRelease 2024-09-14T06:48:57.367 INFO:teuthology.orchestra.run.smithi129.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy Release [7,689 B] 2024-09-14T06:48:57.385 INFO:teuthology.orchestra.run.smithi169.stdout:Get:5 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy Release [7,689 B] 2024-09-14T06:48:57.390 INFO:teuthology.orchestra.run.smithi129.stdout:Ign:3 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-14T06:48:57.407 INFO:teuthology.orchestra.run.smithi169.stdout:Ign:6 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-14T06:48:57.421 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-14T06:48:57.421 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:5 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-14T06:48:57.421 INFO:teuthology.orchestra.run.smithi129.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,420 B] 2024-09-14T06:48:57.428 INFO:teuthology.orchestra.run.smithi169.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,420 B] 2024-09-14T06:48:57.433 INFO:teuthology.orchestra.run.smithi169.stdout:Hit:8 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-14T06:48:57.450 INFO:teuthology.orchestra.run.smithi129.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.2 kB] 2024-09-14T06:48:57.454 INFO:teuthology.orchestra.run.smithi169.stdout:Get:9 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.2 kB] 2024-09-14T06:48:57.493 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-14T06:48:57.575 INFO:teuthology.orchestra.run.smithi129.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-14T06:48:57.662 INFO:teuthology.orchestra.run.smithi169.stdout:Fetched 29.3 kB in 0s (86.9 kB/s) 2024-09-14T06:48:57.799 INFO:teuthology.orchestra.run.smithi129.stdout:Fetched 29.3 kB in 0s (59.0 kB/s) 2024-09-14T06:48:58.665 INFO:teuthology.orchestra.run.smithi169.stdout:Reading package lists... 2024-09-14T06:48:58.681 DEBUG:teuthology.orchestra.run.smithi169:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.3.0-4775-g7f24e589-1jammy cephadm=19.3.0-4775-g7f24e589-1jammy ceph-mds=19.3.0-4775-g7f24e589-1jammy ceph-mgr=19.3.0-4775-g7f24e589-1jammy ceph-common=19.3.0-4775-g7f24e589-1jammy ceph-fuse=19.3.0-4775-g7f24e589-1jammy ceph-test=19.3.0-4775-g7f24e589-1jammy ceph-volume=19.3.0-4775-g7f24e589-1jammy radosgw=19.3.0-4775-g7f24e589-1jammy python3-rados=19.3.0-4775-g7f24e589-1jammy python3-rgw=19.3.0-4775-g7f24e589-1jammy python3-cephfs=19.3.0-4775-g7f24e589-1jammy python3-rbd=19.3.0-4775-g7f24e589-1jammy libcephfs2=19.3.0-4775-g7f24e589-1jammy libcephfs-dev=19.3.0-4775-g7f24e589-1jammy librados2=19.3.0-4775-g7f24e589-1jammy librbd1=19.3.0-4775-g7f24e589-1jammy rbd-fuse=19.3.0-4775-g7f24e589-1jammy rbd-mirror=19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:48:58.745 INFO:teuthology.orchestra.run.smithi169.stdout:Reading package lists... 2024-09-14T06:48:58.768 INFO:teuthology.orchestra.run.smithi129.stdout:Reading package lists... 2024-09-14T06:48:58.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.3.0-4775-g7f24e589-1jammy cephadm=19.3.0-4775-g7f24e589-1jammy ceph-mds=19.3.0-4775-g7f24e589-1jammy ceph-mgr=19.3.0-4775-g7f24e589-1jammy ceph-common=19.3.0-4775-g7f24e589-1jammy ceph-fuse=19.3.0-4775-g7f24e589-1jammy ceph-test=19.3.0-4775-g7f24e589-1jammy ceph-volume=19.3.0-4775-g7f24e589-1jammy radosgw=19.3.0-4775-g7f24e589-1jammy python3-rados=19.3.0-4775-g7f24e589-1jammy python3-rgw=19.3.0-4775-g7f24e589-1jammy python3-cephfs=19.3.0-4775-g7f24e589-1jammy python3-rbd=19.3.0-4775-g7f24e589-1jammy libcephfs2=19.3.0-4775-g7f24e589-1jammy libcephfs-dev=19.3.0-4775-g7f24e589-1jammy librados2=19.3.0-4775-g7f24e589-1jammy librbd1=19.3.0-4775-g7f24e589-1jammy rbd-fuse=19.3.0-4775-g7f24e589-1jammy rbd-mirror=19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:48:58.845 INFO:teuthology.orchestra.run.smithi129.stdout:Reading package lists... 2024-09-14T06:48:58.955 INFO:teuthology.orchestra.run.smithi169.stdout:Building dependency tree... 2024-09-14T06:48:58.956 INFO:teuthology.orchestra.run.smithi169.stdout:Reading state information... 2024-09-14T06:48:59.064 INFO:teuthology.orchestra.run.smithi129.stdout:Building dependency tree... 2024-09-14T06:48:59.065 INFO:teuthology.orchestra.run.smithi129.stdout:Reading state information... 2024-09-14T06:48:59.163 INFO:teuthology.orchestra.run.smithi169.stdout:The following packages were automatically installed and are no longer required: 2024-09-14T06:48:59.163 INFO:teuthology.orchestra.run.smithi169.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-14T06:48:59.163 INFO:teuthology.orchestra.run.smithi169.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-14T06:48:59.164 INFO:teuthology.orchestra.run.smithi169.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-14T06:48:59.164 INFO:teuthology.orchestra.run.smithi169.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-14T06:48:59.164 INFO:teuthology.orchestra.run.smithi169.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-14T06:48:59.164 INFO:teuthology.orchestra.run.smithi169.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-14T06:48:59.165 INFO:teuthology.orchestra.run.smithi169.stdout:The following additional packages will be installed: 2024-09-14T06:48:59.165 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-14T06:48:59.165 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2024-09-14T06:48:59.165 INFO:teuthology.orchestra.run.smithi169.stdout: libdouble-conversion3 libjq1 liblttng-ust1 liblua5.3-dev libnbd0 liboath0 2024-09-14T06:48:59.166 INFO:teuthology.orchestra.run.smithi169.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-14T06:48:59.166 INFO:teuthology.orchestra.run.smithi169.stdout: libradosstriper1 librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph 2024-09-14T06:48:59.166 INFO:teuthology.orchestra.run.smithi169.stdout: libthrift-0.16.0 lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: pkg-config python-asyncssh-doc python-babel-localedata 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-14T06:48:59.167 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat unzip 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: xmlstarlet zip 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout:Suggested packages: 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python3-influxdb readline-doc python-jinja2-doc python3-beaker 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python-mako-doc python-natsort-doc python-openssl-doc python3-openssl-dbg 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit 2024-09-14T06:48:59.168 INFO:teuthology.orchestra.run.smithi169.stdout: python-pecan-doc python-psutil-doc subversion python-pygments-doc 2024-09-14T06:48:59.169 INFO:teuthology.orchestra.run.smithi169.stdout: python-pyinotify-doc python3-socks python-requests-doc python3-dap 2024-09-14T06:48:59.169 INFO:teuthology.orchestra.run.smithi169.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2024-09-14T06:48:59.169 INFO:teuthology.orchestra.run.smithi169.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2024-09-14T06:48:59.169 INFO:teuthology.orchestra.run.smithi169.stdout:Recommended packages: 2024-09-14T06:48:59.169 INFO:teuthology.orchestra.run.smithi169.stdout: btrfs-tools 2024-09-14T06:48:59.269 INFO:teuthology.orchestra.run.smithi129.stdout:The following packages were automatically installed and are no longer required: 2024-09-14T06:48:59.269 INFO:teuthology.orchestra.run.smithi129.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-14T06:48:59.269 INFO:teuthology.orchestra.run.smithi129.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-14T06:48:59.269 INFO:teuthology.orchestra.run.smithi129.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-14T06:48:59.270 INFO:teuthology.orchestra.run.smithi129.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-14T06:48:59.270 INFO:teuthology.orchestra.run.smithi129.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout:The following additional packages will be installed: 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout: libdouble-conversion3 libjq1 liblttng-ust1 liblua5.3-dev libnbd0 liboath0 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-14T06:48:59.271 INFO:teuthology.orchestra.run.smithi129.stdout: libradosstriper1 librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph 2024-09-14T06:48:59.272 INFO:teuthology.orchestra.run.smithi129.stdout: libthrift-0.16.0 lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli 2024-09-14T06:48:59.272 INFO:teuthology.orchestra.run.smithi129.stdout: pkg-config python-asyncssh-doc python-babel-localedata 2024-09-14T06:48:59.272 INFO:teuthology.orchestra.run.smithi129.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-09-14T06:48:59.272 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-14T06:48:59.272 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-14T06:48:59.273 INFO:teuthology.orchestra.run.smithi129.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-14T06:48:59.274 INFO:teuthology.orchestra.run.smithi129.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-14T06:48:59.274 INFO:teuthology.orchestra.run.smithi129.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat unzip 2024-09-14T06:48:59.274 INFO:teuthology.orchestra.run.smithi129.stdout: xmlstarlet zip 2024-09-14T06:48:59.274 INFO:teuthology.orchestra.run.smithi129.stdout:Suggested packages: 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: python3-influxdb readline-doc python-jinja2-doc python3-beaker 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: python-mako-doc python-natsort-doc python-openssl-doc python3-openssl-dbg 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: python-pecan-doc python-psutil-doc subversion python-pygments-doc 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: python-pyinotify-doc python3-socks python-requests-doc python3-dap 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout:Recommended packages: 2024-09-14T06:48:59.275 INFO:teuthology.orchestra.run.smithi129.stdout: btrfs-tools 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout:The following NEW packages will be installed: 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 liblttng-ust1 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: liblua5.3-dev libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: libqt5dbus5 libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2024-09-14T06:48:59.277 INFO:teuthology.orchestra.run.smithi169.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python-babel-localedata python-pastedeploy-tpl python3-asyncssh 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-14T06:48:59.278 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-14T06:48:59.279 INFO:teuthology.orchestra.run.smithi169.stdout: radosgw rbd-fuse rbd-mirror socat unzip xmlstarlet zip 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout:The following NEW packages will be installed: 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 liblttng-ust1 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout: liblua5.3-dev libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2024-09-14T06:48:59.378 INFO:teuthology.orchestra.run.smithi129.stdout: libqt5dbus5 libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2024-09-14T06:48:59.379 INFO:teuthology.orchestra.run.smithi129.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python-babel-localedata python-pastedeploy-tpl python3-asyncssh 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-14T06:48:59.380 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-14T06:48:59.381 INFO:teuthology.orchestra.run.smithi129.stdout: radosgw rbd-fuse rbd-mirror socat unzip xmlstarlet zip 2024-09-14T06:48:59.418 INFO:teuthology.orchestra.run.smithi169.stdout:0 upgraded, 120 newly installed, 0 to remove and 331 not upgraded. 2024-09-14T06:48:59.418 INFO:teuthology.orchestra.run.smithi169.stdout:Need to get 254 MB of archives. 2024-09-14T06:48:59.418 INFO:teuthology.orchestra.run.smithi169.stdout:After this operation, 947 MB of additional disk space will be used. 2024-09-14T06:48:59.418 INFO:teuthology.orchestra.run.smithi169.stdout:Get:1 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.3.0-4775-g7f24e589-1jammy [3,631 kB] 2024-09-14T06:48:59.439 INFO:teuthology.orchestra.run.smithi129.stdout:0 upgraded, 120 newly installed, 0 to remove and 331 not upgraded. 2024-09-14T06:48:59.439 INFO:teuthology.orchestra.run.smithi129.stdout:Need to get 254 MB of archives. 2024-09-14T06:48:59.439 INFO:teuthology.orchestra.run.smithi129.stdout:After this operation, 947 MB of additional disk space will be used. 2024-09-14T06:48:59.439 INFO:teuthology.orchestra.run.smithi129.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-14T06:48:59.463 INFO:teuthology.orchestra.run.smithi169.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-14T06:48:59.521 INFO:teuthology.orchestra.run.smithi129.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.3.0-4775-g7f24e589-1jammy [3,631 kB] 2024-09-14T06:48:59.526 INFO:teuthology.orchestra.run.smithi129.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-14T06:48:59.529 INFO:teuthology.orchestra.run.smithi129.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-14T06:48:59.548 INFO:teuthology.orchestra.run.smithi129.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-14T06:48:59.601 INFO:teuthology.orchestra.run.smithi169.stdout:Get:3 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.3.0-4775-g7f24e589-1jammy [3,330 kB] 2024-09-14T06:48:59.604 INFO:teuthology.orchestra.run.smithi129.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-14T06:48:59.608 INFO:teuthology.orchestra.run.smithi129.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-14T06:48:59.621 INFO:teuthology.orchestra.run.smithi129.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-14T06:48:59.625 INFO:teuthology.orchestra.run.smithi129.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2024-09-14T06:48:59.626 INFO:teuthology.orchestra.run.smithi129.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-14T06:48:59.627 INFO:teuthology.orchestra.run.smithi129.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-14T06:48:59.628 INFO:teuthology.orchestra.run.smithi129.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-14T06:48:59.636 INFO:teuthology.orchestra.run.smithi129.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2024-09-14T06:48:59.638 INFO:teuthology.orchestra.run.smithi129.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2024-09-14T06:48:59.640 INFO:teuthology.orchestra.run.smithi129.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2024-09-14T06:48:59.642 INFO:teuthology.orchestra.run.smithi129.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5,034 B] 2024-09-14T06:48:59.642 INFO:teuthology.orchestra.run.smithi129.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2024-09-14T06:48:59.659 INFO:teuthology.orchestra.run.smithi129.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2024-09-14T06:48:59.661 INFO:teuthology.orchestra.run.smithi129.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2024-09-14T06:48:59.662 INFO:teuthology.orchestra.run.smithi169.stdout:Get:4 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.3.0-4775-g7f24e589-1jammy [722 kB] 2024-09-14T06:48:59.663 INFO:teuthology.orchestra.run.smithi129.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-14T06:48:59.663 INFO:teuthology.orchestra.run.smithi129.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-14T06:48:59.663 INFO:teuthology.orchestra.run.smithi129.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-14T06:48:59.665 INFO:teuthology.orchestra.run.smithi129.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-14T06:48:59.665 INFO:teuthology.orchestra.run.smithi129.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-14T06:48:59.673 INFO:teuthology.orchestra.run.smithi169.stdout:Get:5 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.3.0-4775-g7f24e589-1jammy [356 kB] 2024-09-14T06:48:59.675 INFO:teuthology.orchestra.run.smithi129.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-14T06:48:59.676 INFO:teuthology.orchestra.run.smithi129.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-14T06:48:59.676 INFO:teuthology.orchestra.run.smithi129.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-14T06:48:59.688 INFO:teuthology.orchestra.run.smithi169.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.3.0-4775-g7f24e589-1jammy [32.8 kB] 2024-09-14T06:48:59.688 INFO:teuthology.orchestra.run.smithi169.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.3.0-4775-g7f24e589-1jammy [180 kB] 2024-09-14T06:48:59.691 INFO:teuthology.orchestra.run.smithi169.stdout:Get:8 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.3.0-4775-g7f24e589-1jammy [71.2 kB] 2024-09-14T06:48:59.692 INFO:teuthology.orchestra.run.smithi169.stdout:Get:9 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.3.0-4775-g7f24e589-1jammy [338 kB] 2024-09-14T06:48:59.692 INFO:teuthology.orchestra.run.smithi129.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-14T06:48:59.693 INFO:teuthology.orchestra.run.smithi129.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-14T06:48:59.697 INFO:teuthology.orchestra.run.smithi169.stdout:Get:10 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.3.0-4775-g7f24e589-1jammy [7,984 kB] 2024-09-14T06:48:59.697 INFO:teuthology.orchestra.run.smithi129.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-14T06:48:59.698 INFO:teuthology.orchestra.run.smithi129.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-14T06:48:59.699 INFO:teuthology.orchestra.run.smithi129.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-14T06:48:59.700 INFO:teuthology.orchestra.run.smithi129.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-14T06:48:59.701 INFO:teuthology.orchestra.run.smithi129.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-14T06:48:59.709 INFO:teuthology.orchestra.run.smithi129.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-14T06:48:59.710 INFO:teuthology.orchestra.run.smithi129.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-14T06:48:59.712 INFO:teuthology.orchestra.run.smithi129.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-14T06:48:59.726 INFO:teuthology.orchestra.run.smithi129.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-14T06:48:59.727 INFO:teuthology.orchestra.run.smithi129.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-14T06:48:59.735 INFO:teuthology.orchestra.run.smithi129.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-14T06:48:59.736 INFO:teuthology.orchestra.run.smithi129.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-14T06:48:59.737 INFO:teuthology.orchestra.run.smithi129.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-14T06:48:59.738 INFO:teuthology.orchestra.run.smithi129.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-14T06:48:59.740 INFO:teuthology.orchestra.run.smithi129.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-14T06:48:59.743 INFO:teuthology.orchestra.run.smithi129.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-14T06:48:59.744 INFO:teuthology.orchestra.run.smithi129.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-14T06:48:59.747 INFO:teuthology.orchestra.run.smithi129.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-14T06:48:59.760 INFO:teuthology.orchestra.run.smithi129.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-14T06:48:59.764 INFO:teuthology.orchestra.run.smithi129.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-14T06:48:59.765 INFO:teuthology.orchestra.run.smithi129.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-14T06:48:59.766 INFO:teuthology.orchestra.run.smithi129.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-09-14T06:48:59.770 INFO:teuthology.orchestra.run.smithi129.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-14T06:48:59.833 INFO:teuthology.orchestra.run.smithi129.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-14T06:48:59.834 INFO:teuthology.orchestra.run.smithi129.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-14T06:48:59.835 INFO:teuthology.orchestra.run.smithi129.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-14T06:48:59.846 INFO:teuthology.orchestra.run.smithi129.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-14T06:48:59.847 INFO:teuthology.orchestra.run.smithi129.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-14T06:48:59.851 INFO:teuthology.orchestra.run.smithi129.stdout:Get:58 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.3.0-4775-g7f24e589-1jammy [3,330 kB] 2024-09-14T06:48:59.864 INFO:teuthology.orchestra.run.smithi129.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-14T06:48:59.866 INFO:teuthology.orchestra.run.smithi129.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-14T06:48:59.866 INFO:teuthology.orchestra.run.smithi129.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-14T06:48:59.882 INFO:teuthology.orchestra.run.smithi129.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-14T06:48:59.882 INFO:teuthology.orchestra.run.smithi129.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-14T06:48:59.883 INFO:teuthology.orchestra.run.smithi129.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-14T06:48:59.883 INFO:teuthology.orchestra.run.smithi129.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-14T06:48:59.883 INFO:teuthology.orchestra.run.smithi129.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-14T06:48:59.884 INFO:teuthology.orchestra.run.smithi129.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-14T06:48:59.884 INFO:teuthology.orchestra.run.smithi169.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-14T06:48:59.885 INFO:teuthology.orchestra.run.smithi169.stdout:Get:12 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.3.0-4775-g7f24e589-1jammy [112 kB] 2024-09-14T06:48:59.886 INFO:teuthology.orchestra.run.smithi129.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-14T06:48:59.886 INFO:teuthology.orchestra.run.smithi129.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-14T06:48:59.887 INFO:teuthology.orchestra.run.smithi169.stdout:Get:13 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.3.0-4775-g7f24e589-1jammy [473 kB] 2024-09-14T06:48:59.889 INFO:teuthology.orchestra.run.smithi129.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-14T06:48:59.894 INFO:teuthology.orchestra.run.smithi169.stdout:Get:14 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.3.0-4775-g7f24e589-1jammy [28.3 MB] 2024-09-14T06:48:59.896 INFO:teuthology.orchestra.run.smithi169.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-14T06:48:59.902 INFO:teuthology.orchestra.run.smithi129.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-14T06:48:59.904 INFO:teuthology.orchestra.run.smithi129.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-14T06:48:59.904 INFO:teuthology.orchestra.run.smithi129.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-14T06:48:59.908 INFO:teuthology.orchestra.run.smithi129.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-14T06:48:59.910 INFO:teuthology.orchestra.run.smithi129.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2024-09-14T06:48:59.911 INFO:teuthology.orchestra.run.smithi129.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2024-09-14T06:48:59.912 INFO:teuthology.orchestra.run.smithi129.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-14T06:48:59.920 INFO:teuthology.orchestra.run.smithi129.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2024-09-14T06:48:59.920 INFO:teuthology.orchestra.run.smithi129.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-09-14T06:48:59.923 INFO:teuthology.orchestra.run.smithi129.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-14T06:48:59.936 INFO:teuthology.orchestra.run.smithi129.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-14T06:48:59.937 INFO:teuthology.orchestra.run.smithi129.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-14T06:48:59.937 INFO:teuthology.orchestra.run.smithi129.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-14T06:48:59.939 INFO:teuthology.orchestra.run.smithi129.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-14T06:48:59.940 INFO:teuthology.orchestra.run.smithi129.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-14T06:48:59.946 INFO:teuthology.orchestra.run.smithi129.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-14T06:48:59.946 INFO:teuthology.orchestra.run.smithi129.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-14T06:48:59.953 INFO:teuthology.orchestra.run.smithi129.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-14T06:48:59.955 INFO:teuthology.orchestra.run.smithi129.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-14T06:48:59.956 INFO:teuthology.orchestra.run.smithi129.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-14T06:48:59.977 INFO:teuthology.orchestra.run.smithi169.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-14T06:49:00.045 INFO:teuthology.orchestra.run.smithi129.stdout:Get:91 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.3.0-4775-g7f24e589-1jammy [722 kB] 2024-09-14T06:49:00.094 INFO:teuthology.orchestra.run.smithi129.stdout:Get:92 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.3.0-4775-g7f24e589-1jammy [356 kB] 2024-09-14T06:49:00.140 INFO:teuthology.orchestra.run.smithi129.stdout:Get:93 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.3.0-4775-g7f24e589-1jammy [32.8 kB] 2024-09-14T06:49:00.141 INFO:teuthology.orchestra.run.smithi129.stdout:Get:94 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.3.0-4775-g7f24e589-1jammy [180 kB] 2024-09-14T06:49:00.144 INFO:teuthology.orchestra.run.smithi129.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.3.0-4775-g7f24e589-1jammy [71.2 kB] 2024-09-14T06:49:00.146 INFO:teuthology.orchestra.run.smithi129.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.3.0-4775-g7f24e589-1jammy [338 kB] 2024-09-14T06:49:00.153 INFO:teuthology.orchestra.run.smithi129.stdout:Get:97 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.3.0-4775-g7f24e589-1jammy [7,984 kB] 2024-09-14T06:49:00.207 INFO:teuthology.orchestra.run.smithi169.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-14T06:49:00.217 INFO:teuthology.orchestra.run.smithi169.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-14T06:49:00.247 INFO:teuthology.orchestra.run.smithi169.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-14T06:49:00.255 INFO:teuthology.orchestra.run.smithi169.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2024-09-14T06:49:00.258 INFO:teuthology.orchestra.run.smithi169.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-14T06:49:00.259 INFO:teuthology.orchestra.run.smithi169.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-14T06:49:00.260 INFO:teuthology.orchestra.run.smithi169.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-14T06:49:00.278 INFO:teuthology.orchestra.run.smithi169.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2024-09-14T06:49:00.282 INFO:teuthology.orchestra.run.smithi169.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2024-09-14T06:49:00.288 INFO:teuthology.orchestra.run.smithi169.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2024-09-14T06:49:00.369 INFO:teuthology.orchestra.run.smithi169.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5,034 B] 2024-09-14T06:49:00.369 INFO:teuthology.orchestra.run.smithi169.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2024-09-14T06:49:00.374 INFO:teuthology.orchestra.run.smithi169.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2024-09-14T06:49:00.378 INFO:teuthology.orchestra.run.smithi169.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2024-09-14T06:49:00.381 INFO:teuthology.orchestra.run.smithi169.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-14T06:49:00.382 INFO:teuthology.orchestra.run.smithi169.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-14T06:49:00.383 INFO:teuthology.orchestra.run.smithi169.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-14T06:49:00.385 INFO:teuthology.orchestra.run.smithi169.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-14T06:49:00.385 INFO:teuthology.orchestra.run.smithi169.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-14T06:49:00.452 INFO:teuthology.orchestra.run.smithi169.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-14T06:49:00.468 INFO:teuthology.orchestra.run.smithi169.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-14T06:49:00.469 INFO:teuthology.orchestra.run.smithi169.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-14T06:49:00.470 INFO:teuthology.orchestra.run.smithi169.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-14T06:49:00.471 INFO:teuthology.orchestra.run.smithi169.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-14T06:49:00.476 INFO:teuthology.orchestra.run.smithi169.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-14T06:49:00.477 INFO:teuthology.orchestra.run.smithi169.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-14T06:49:00.561 INFO:teuthology.orchestra.run.smithi169.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-14T06:49:00.562 INFO:teuthology.orchestra.run.smithi169.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-14T06:49:00.605 INFO:teuthology.orchestra.run.smithi129.stdout:Get:98 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.3.0-4775-g7f24e589-1jammy [112 kB] 2024-09-14T06:49:00.616 INFO:teuthology.orchestra.run.smithi169.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-14T06:49:00.617 INFO:teuthology.orchestra.run.smithi169.stdout:Get:46 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.3.0-4775-g7f24e589-1jammy [5,206 kB] 2024-09-14T06:49:00.617 INFO:teuthology.orchestra.run.smithi169.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-14T06:49:00.618 INFO:teuthology.orchestra.run.smithi169.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-14T06:49:00.621 INFO:teuthology.orchestra.run.smithi169.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-14T06:49:00.623 INFO:teuthology.orchestra.run.smithi169.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-14T06:49:00.623 INFO:teuthology.orchestra.run.smithi169.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-14T06:49:00.634 INFO:teuthology.orchestra.run.smithi169.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-14T06:49:00.640 INFO:teuthology.orchestra.run.smithi129.stdout:Get:99 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.3.0-4775-g7f24e589-1jammy [473 kB] 2024-09-14T06:49:00.650 INFO:teuthology.orchestra.run.smithi129.stdout:Get:100 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.3.0-4775-g7f24e589-1jammy [28.3 MB] 2024-09-14T06:49:00.684 INFO:teuthology.orchestra.run.smithi169.stdout:Get:53 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.3.0-4775-g7f24e589-1jammy [251 kB] 2024-09-14T06:49:00.687 INFO:teuthology.orchestra.run.smithi169.stdout:Get:54 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.3.0-4775-g7f24e589-1jammy [124 kB] 2024-09-14T06:49:00.688 INFO:teuthology.orchestra.run.smithi169.stdout:Get:55 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.3.0-4775-g7f24e589-1jammy [1,561 kB] 2024-09-14T06:49:00.696 INFO:teuthology.orchestra.run.smithi169.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-14T06:49:00.716 INFO:teuthology.orchestra.run.smithi169.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-14T06:49:00.717 INFO:teuthology.orchestra.run.smithi169.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-14T06:49:00.719 INFO:teuthology.orchestra.run.smithi169.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-14T06:49:00.723 INFO:teuthology.orchestra.run.smithi169.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-14T06:49:00.735 INFO:teuthology.orchestra.run.smithi169.stdout:Get:61 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.3.0-4775-g7f24e589-1jammy [6,311 kB] 2024-09-14T06:49:00.779 INFO:teuthology.orchestra.run.smithi169.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-14T06:49:00.782 INFO:teuthology.orchestra.run.smithi169.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-14T06:49:00.784 INFO:teuthology.orchestra.run.smithi169.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-14T06:49:00.788 INFO:teuthology.orchestra.run.smithi169.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-14T06:49:00.790 INFO:teuthology.orchestra.run.smithi169.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-14T06:49:00.860 INFO:teuthology.orchestra.run.smithi169.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-09-14T06:49:00.864 INFO:teuthology.orchestra.run.smithi169.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-14T06:49:00.889 INFO:teuthology.orchestra.run.smithi169.stdout:Get:69 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.3.0-4775-g7f24e589-1jammy [23.1 MB] 2024-09-14T06:49:01.029 INFO:teuthology.orchestra.run.smithi169.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-14T06:49:01.031 INFO:teuthology.orchestra.run.smithi169.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-14T06:49:01.034 INFO:teuthology.orchestra.run.smithi169.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-14T06:49:01.035 INFO:teuthology.orchestra.run.smithi169.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-14T06:49:01.037 INFO:teuthology.orchestra.run.smithi169.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-14T06:49:01.073 INFO:teuthology.orchestra.run.smithi169.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-14T06:49:01.076 INFO:teuthology.orchestra.run.smithi169.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-14T06:49:01.077 INFO:teuthology.orchestra.run.smithi169.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-14T06:49:01.136 INFO:teuthology.orchestra.run.smithi169.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-14T06:49:01.136 INFO:teuthology.orchestra.run.smithi169.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-14T06:49:01.198 INFO:teuthology.orchestra.run.smithi169.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-14T06:49:01.199 INFO:teuthology.orchestra.run.smithi169.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-14T06:49:01.200 INFO:teuthology.orchestra.run.smithi169.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-14T06:49:01.201 INFO:teuthology.orchestra.run.smithi169.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-14T06:49:01.201 INFO:teuthology.orchestra.run.smithi169.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-14T06:49:01.202 INFO:teuthology.orchestra.run.smithi169.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-14T06:49:01.209 INFO:teuthology.orchestra.run.smithi169.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-14T06:49:01.279 INFO:teuthology.orchestra.run.smithi169.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-14T06:49:01.282 INFO:teuthology.orchestra.run.smithi169.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-14T06:49:01.283 INFO:teuthology.orchestra.run.smithi169.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-14T06:49:01.362 INFO:teuthology.orchestra.run.smithi169.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-14T06:49:01.369 INFO:teuthology.orchestra.run.smithi169.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2024-09-14T06:49:01.371 INFO:teuthology.orchestra.run.smithi169.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2024-09-14T06:49:01.372 INFO:teuthology.orchestra.run.smithi169.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-14T06:49:01.383 INFO:teuthology.orchestra.run.smithi169.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2024-09-14T06:49:01.385 INFO:teuthology.orchestra.run.smithi169.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-09-14T06:49:01.392 INFO:teuthology.orchestra.run.smithi169.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-14T06:49:01.446 INFO:teuthology.orchestra.run.smithi169.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-14T06:49:01.520 INFO:teuthology.orchestra.run.smithi169.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-14T06:49:01.520 INFO:teuthology.orchestra.run.smithi169.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-14T06:49:01.521 INFO:teuthology.orchestra.run.smithi169.stdout:Get:100 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.3.0-4775-g7f24e589-1jammy [14.1 kB] 2024-09-14T06:49:01.522 INFO:teuthology.orchestra.run.smithi169.stdout:Get:101 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.3.0-4775-g7f24e589-1jammy [880 kB] 2024-09-14T06:49:01.527 INFO:teuthology.orchestra.run.smithi169.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-14T06:49:01.529 INFO:teuthology.orchestra.run.smithi169.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-14T06:49:01.536 INFO:teuthology.orchestra.run.smithi169.stdout:Get:104 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.3.0-4775-g7f24e589-1jammy [2,452 kB] 2024-09-14T06:49:01.546 INFO:teuthology.orchestra.run.smithi169.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-14T06:49:01.547 INFO:teuthology.orchestra.run.smithi169.stdout:Get:106 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-14T06:49:01.548 INFO:teuthology.orchestra.run.smithi169.stdout:Get:107 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-14T06:49:01.553 INFO:teuthology.orchestra.run.smithi169.stdout:Get:108 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-14T06:49:01.571 INFO:teuthology.orchestra.run.smithi169.stdout:Get:109 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-14T06:49:01.574 INFO:teuthology.orchestra.run.smithi169.stdout:Get:110 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.3.0-4775-g7f24e589-1jammy [812 kB] 2024-09-14T06:49:01.583 INFO:teuthology.orchestra.run.smithi169.stdout:Get:111 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.3.0-4775-g7f24e589-1jammy [160 kB] 2024-09-14T06:49:01.585 INFO:teuthology.orchestra.run.smithi169.stdout:Get:112 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.3.0-4775-g7f24e589-1jammy [61.4 MB] 2024-09-14T06:49:02.065 INFO:teuthology.orchestra.run.smithi129.stdout:Get:101 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.3.0-4775-g7f24e589-1jammy [5,206 kB] 2024-09-14T06:49:02.298 INFO:teuthology.orchestra.run.smithi129.stdout:Get:102 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.3.0-4775-g7f24e589-1jammy [251 kB] 2024-09-14T06:49:02.303 INFO:teuthology.orchestra.run.smithi129.stdout:Get:103 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.3.0-4775-g7f24e589-1jammy [124 kB] 2024-09-14T06:49:02.306 INFO:teuthology.orchestra.run.smithi129.stdout:Get:104 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.3.0-4775-g7f24e589-1jammy [1,561 kB] 2024-09-14T06:49:02.335 INFO:teuthology.orchestra.run.smithi129.stdout:Get:105 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.3.0-4775-g7f24e589-1jammy [6,311 kB] 2024-09-14T06:49:02.579 INFO:teuthology.orchestra.run.smithi129.stdout:Get:106 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.3.0-4775-g7f24e589-1jammy [23.1 MB] 2024-09-14T06:49:03.411 INFO:teuthology.orchestra.run.smithi169.stdout:Get:113 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.3.0-4775-g7f24e589-1jammy [8,625 kB] 2024-09-14T06:49:03.486 INFO:teuthology.orchestra.run.smithi129.stdout:Get:107 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.3.0-4775-g7f24e589-1jammy [14.1 kB] 2024-09-14T06:49:03.486 INFO:teuthology.orchestra.run.smithi129.stdout:Get:108 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.3.0-4775-g7f24e589-1jammy [880 kB] 2024-09-14T06:49:03.501 INFO:teuthology.orchestra.run.smithi129.stdout:Get:109 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.3.0-4775-g7f24e589-1jammy [2,452 kB] 2024-09-14T06:49:03.541 INFO:teuthology.orchestra.run.smithi169.stdout:Get:114 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.3.0-4775-g7f24e589-1jammy [14.2 kB] 2024-09-14T06:49:03.541 INFO:teuthology.orchestra.run.smithi169.stdout:Get:115 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.3.0-4775-g7f24e589-1jammy [55.8 MB] 2024-09-14T06:49:03.567 INFO:teuthology.orchestra.run.smithi129.stdout:Get:110 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.3.0-4775-g7f24e589-1jammy [812 kB] 2024-09-14T06:49:03.580 INFO:teuthology.orchestra.run.smithi129.stdout:Get:111 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.3.0-4775-g7f24e589-1jammy [160 kB] 2024-09-14T06:49:03.583 INFO:teuthology.orchestra.run.smithi129.stdout:Get:112 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.3.0-4775-g7f24e589-1jammy [61.4 MB] 2024-09-14T06:49:05.360 INFO:teuthology.orchestra.run.smithi169.stdout:Get:116 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.3.0-4775-g7f24e589-1jammy [130 kB] 2024-09-14T06:49:05.363 INFO:teuthology.orchestra.run.smithi169.stdout:Get:117 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.3.0-4775-g7f24e589-1jammy [39.0 kB] 2024-09-14T06:49:05.364 INFO:teuthology.orchestra.run.smithi169.stdout:Get:118 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.3.0-4775-g7f24e589-1jammy [15.9 MB] 2024-09-14T06:49:05.804 INFO:teuthology.orchestra.run.smithi169.stdout:Get:119 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.3.0-4775-g7f24e589-1jammy [91.3 kB] 2024-09-14T06:49:05.806 INFO:teuthology.orchestra.run.smithi169.stdout:Get:120 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.3.0-4775-g7f24e589-1jammy [3,225 kB] 2024-09-14T06:49:06.067 INFO:teuthology.orchestra.run.smithi129.stdout:Get:113 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.3.0-4775-g7f24e589-1jammy [8,625 kB] 2024-09-14T06:49:06.181 INFO:teuthology.orchestra.run.smithi129.stdout:Get:114 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.3.0-4775-g7f24e589-1jammy [14.2 kB] 2024-09-14T06:49:06.181 INFO:teuthology.orchestra.run.smithi129.stdout:Get:115 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.3.0-4775-g7f24e589-1jammy [55.8 MB] 2024-09-14T06:49:06.457 INFO:teuthology.orchestra.run.smithi169.stdout:Fetched 254 MB in 7s (38.7 MB/s) 2024-09-14T06:49:06.646 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-14T06:49:06.747 INFO:teuthology.orchestra.run.smithi169.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-14T06:49:06.752 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-14T06:49:06.836 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-14T06:49:07.173 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-14T06:49:07.199 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-14T06:49:07.239 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-14T06:49:07.394 INFO:teuthology.orchestra.run.smithi129.stdout:Get:116 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.3.0-4775-g7f24e589-1jammy [130 kB] 2024-09-14T06:49:07.397 INFO:teuthology.orchestra.run.smithi129.stdout:Get:117 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.3.0-4775-g7f24e589-1jammy [39.0 kB] 2024-09-14T06:49:07.398 INFO:teuthology.orchestra.run.smithi129.stdout:Get:118 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.3.0-4775-g7f24e589-1jammy [15.9 MB] 2024-09-14T06:49:07.501 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-14T06:49:07.516 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-14T06:49:07.551 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-14T06:49:07.740 INFO:teuthology.orchestra.run.smithi129.stdout:Get:119 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.3.0-4775-g7f24e589-1jammy [91.3 kB] 2024-09-14T06:49:07.741 INFO:teuthology.orchestra.run.smithi129.stdout:Get:120 https://4.chacra.ceph.com/r/ceph/main/7f24e589328ae9e639c0009d93340e5b235d0a47/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.3.0-4775-g7f24e589-1jammy [3,225 kB] 2024-09-14T06:49:08.013 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-14T06:49:08.039 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-14T06:49:08.087 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:49:08.374 INFO:teuthology.orchestra.run.smithi129.stdout:Fetched 254 MB in 8s (30.2 MB/s) 2024-09-14T06:49:08.499 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-14T06:49:08.526 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-14T06:49:08.548 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-14T06:49:08.565 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:49:08.649 INFO:teuthology.orchestra.run.smithi129.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-14T06:49:08.655 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-14T06:49:08.747 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-14T06:49:08.877 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-14T06:49:08.904 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-14T06:49:08.935 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:49:09.109 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-14T06:49:09.135 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-14T06:49:09.175 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-14T06:49:09.238 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-14T06:49:09.265 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-14T06:49:09.305 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-14T06:49:09.470 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-14T06:49:09.496 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-14T06:49:09.528 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-14T06:49:09.599 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package librados2. 2024-09-14T06:49:09.625 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../007-librados2_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:09.666 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking librados2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:09.974 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-14T06:49:10.000 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-14T06:49:10.048 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:49:10.077 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libnbd0. 2024-09-14T06:49:10.103 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../008-libnbd0_1.10.5-1_amd64.deb ... 2024-09-14T06:49:10.136 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libnbd0 (1.10.5-1) ... 2024-09-14T06:49:10.422 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package librbd1. 2024-09-14T06:49:10.448 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../009-librbd1_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:10.452 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-14T06:49:10.478 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-14T06:49:10.488 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking librbd1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:10.518 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:49:10.822 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-14T06:49:10.849 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-14T06:49:10.880 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:49:10.941 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libcephfs2. 2024-09-14T06:49:10.967 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../010-libcephfs2_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:10.999 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libcephfs2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:11.225 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-14T06:49:11.251 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-14T06:49:11.269 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-rados. 2024-09-14T06:49:11.291 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-14T06:49:11.295 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../011-python3-rados_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:11.335 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-rados (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:11.636 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package librados2. 2024-09-14T06:49:11.655 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-14T06:49:11.663 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../007-librados2_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:11.681 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../012-python3-ceph-argparse_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:11.702 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking librados2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:11.721 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-ceph-argparse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:11.974 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-cephfs. 2024-09-14T06:49:11.989 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../013-python3-cephfs_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:12.032 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-cephfs (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:12.156 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libnbd0. 2024-09-14T06:49:12.183 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../008-libnbd0_1.10.5-1_amd64.deb ... 2024-09-14T06:49:12.214 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libnbd0 (1.10.5-1) ... 2024-09-14T06:49:12.334 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-14T06:49:12.361 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../014-python3-ceph-common_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:12.410 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:12.550 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package librbd1. 2024-09-14T06:49:12.577 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../009-librbd1_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:12.617 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking librbd1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:12.738 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-14T06:49:12.764 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-14T06:49:12.804 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-14T06:49:13.028 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libcephfs2. 2024-09-14T06:49:13.054 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../010-libcephfs2_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:13.095 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libcephfs2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:13.124 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-prettytable. 2024-09-14T06:49:13.150 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-09-14T06:49:13.190 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-14T06:49:13.381 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-rados. 2024-09-14T06:49:13.397 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../011-python3-rados_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:13.426 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-rbd. 2024-09-14T06:49:13.439 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-rados (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:13.442 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../017-python3-rbd_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:13.484 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-rbd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:13.759 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-14T06:49:13.780 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-14T06:49:13.785 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../012-python3-ceph-argparse_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:13.809 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-14T06:49:13.825 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-ceph-argparse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:13.846 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-14T06:49:14.061 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-cephfs. 2024-09-14T06:49:14.087 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../013-python3-cephfs_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:14.120 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-cephfs (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:14.123 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libreadline-dev:amd64. 2024-09-14T06:49:14.150 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../019-libreadline-dev_8.1.2-1_amd64.deb ... 2024-09-14T06:49:14.190 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2024-09-14T06:49:14.423 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-14T06:49:14.449 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../014-python3-ceph-common_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:14.485 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2024-09-14T06:49:14.489 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:14.510 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../020-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2024-09-14T06:49:14.551 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-14T06:49:14.809 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-14T06:49:14.824 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-14T06:49:14.867 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-14T06:49:14.905 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package lua5.1. 2024-09-14T06:49:14.931 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../021-lua5.1_5.1.5-8.1build4_amd64.deb ... 2024-09-14T06:49:14.971 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2024-09-14T06:49:15.187 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-prettytable. 2024-09-14T06:49:15.214 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-09-14T06:49:15.253 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-14T06:49:15.282 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package lua-any. 2024-09-14T06:49:15.298 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../022-lua-any_27ubuntu1_all.deb ... 2024-09-14T06:49:15.340 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking lua-any (27ubuntu1) ... 2024-09-14T06:49:15.525 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-rbd. 2024-09-14T06:49:15.551 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../017-python3-rbd_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:15.585 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package zip. 2024-09-14T06:49:15.591 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-rbd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:15.600 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../023-zip_3.0-12build2_amd64.deb ... 2024-09-14T06:49:15.652 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking zip (3.0-12build2) ... 2024-09-14T06:49:15.973 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-14T06:49:15.996 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-14T06:49:15.996 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package unzip. 2024-09-14T06:49:16.023 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../024-unzip_6.0-26ubuntu3.2_amd64.deb ... 2024-09-14T06:49:16.044 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-14T06:49:16.063 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2024-09-14T06:49:16.372 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libreadline-dev:amd64. 2024-09-14T06:49:16.398 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../019-libreadline-dev_8.1.2-1_amd64.deb ... 2024-09-14T06:49:16.417 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package luarocks. 2024-09-14T06:49:16.432 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../025-luarocks_3.8.0+dfsg1-1_all.deb ... 2024-09-14T06:49:16.446 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2024-09-14T06:49:16.466 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2024-09-14T06:49:16.775 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2024-09-14T06:49:16.790 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../020-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2024-09-14T06:49:16.833 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-14T06:49:16.858 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package librgw2. 2024-09-14T06:49:16.884 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../026-librgw2_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:16.924 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking librgw2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:17.161 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package lua5.1. 2024-09-14T06:49:17.188 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../021-lua5.1_5.1.5-8.1build4_amd64.deb ... 2024-09-14T06:49:17.227 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2024-09-14T06:49:17.519 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-rgw. 2024-09-14T06:49:17.546 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../027-python3-rgw_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:17.585 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-rgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:17.589 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package lua-any. 2024-09-14T06:49:17.615 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../022-lua-any_27ubuntu1_all.deb ... 2024-09-14T06:49:17.664 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking lua-any (27ubuntu1) ... 2024-09-14T06:49:17.922 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-14T06:49:17.948 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../028-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-14T06:49:17.988 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-14T06:49:18.175 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package zip. 2024-09-14T06:49:18.190 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../023-zip_3.0-12build2_amd64.deb ... 2024-09-14T06:49:18.233 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking zip (3.0-12build2) ... 2024-09-14T06:49:18.274 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libradosstriper1. 2024-09-14T06:49:18.301 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../029-libradosstriper1_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:18.341 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libradosstriper1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:18.545 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package unzip. 2024-09-14T06:49:18.571 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../024-unzip_6.0-26ubuntu3.2_amd64.deb ... 2024-09-14T06:49:18.611 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2024-09-14T06:49:18.761 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-common. 2024-09-14T06:49:18.788 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../030-ceph-common_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:18.827 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:18.998 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package luarocks. 2024-09-14T06:49:19.025 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../025-luarocks_3.8.0+dfsg1-1_all.deb ... 2024-09-14T06:49:19.090 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2024-09-14T06:49:19.481 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package librgw2. 2024-09-14T06:49:19.508 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../026-librgw2_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:19.547 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking librgw2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:20.158 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-rgw. 2024-09-14T06:49:20.174 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../027-python3-rgw_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:20.208 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-rgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:20.324 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-base. 2024-09-14T06:49:20.351 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../031-ceph-base_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:20.400 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-base (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:20.553 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-14T06:49:20.579 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../028-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-14T06:49:20.619 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-14T06:49:20.930 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libradosstriper1. 2024-09-14T06:49:20.957 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../029-libradosstriper1_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:20.981 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-14T06:49:20.997 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libradosstriper1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:21.007 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../032-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-14T06:49:21.047 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-14T06:49:21.325 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-cheroot. 2024-09-14T06:49:21.351 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../033-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-14T06:49:21.392 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-14T06:49:21.400 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-common. 2024-09-14T06:49:21.427 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../030-ceph-common_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:21.467 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:21.953 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-14T06:49:21.968 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../034-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-14T06:49:22.003 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-14T06:49:22.306 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-14T06:49:22.333 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../035-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-14T06:49:22.373 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-14T06:49:22.684 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-14T06:49:22.711 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../036-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-14T06:49:22.750 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-14T06:49:22.972 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-base. 2024-09-14T06:49:22.999 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../031-ceph-base_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:23.039 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-base (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:23.095 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-tempora. 2024-09-14T06:49:23.121 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../037-python3-tempora_4.1.2-1_all.deb ... 2024-09-14T06:49:23.162 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-14T06:49:23.448 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-portend. 2024-09-14T06:49:23.475 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../038-python3-portend_3.0.0-1_all.deb ... 2024-09-14T06:49:23.515 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-14T06:49:23.768 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-14T06:49:23.793 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-14T06:49:23.794 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../032-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-14T06:49:23.819 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../039-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-14T06:49:23.834 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-14T06:49:23.859 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-14T06:49:24.137 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-cheroot. 2024-09-14T06:49:24.162 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-14T06:49:24.163 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../033-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-14T06:49:24.188 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../040-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-14T06:49:24.203 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-14T06:49:24.228 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-14T06:49:24.523 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-14T06:49:24.549 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../034-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-14T06:49:24.558 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-natsort. 2024-09-14T06:49:24.584 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../041-python3-natsort_8.0.2-1_all.deb ... 2024-09-14T06:49:24.590 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-14T06:49:24.641 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-14T06:49:24.851 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-14T06:49:24.877 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../035-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-14T06:49:24.909 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-14T06:49:24.936 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-logutils. 2024-09-14T06:49:24.962 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../042-python3-logutils_0.3.3-8_all.deb ... 2024-09-14T06:49:25.002 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-14T06:49:25.197 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-14T06:49:25.223 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../036-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-14T06:49:25.265 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-14T06:49:25.298 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-14T06:49:25.324 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../043-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-14T06:49:25.364 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-14T06:49:25.565 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-tempora. 2024-09-14T06:49:25.591 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../037-python3-tempora_4.1.2-1_all.deb ... 2024-09-14T06:49:25.631 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-14T06:49:25.666 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-mako. 2024-09-14T06:49:25.693 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../044-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-14T06:49:25.724 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-14T06:49:25.951 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-portend. 2024-09-14T06:49:25.977 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../038-python3-portend_3.0.0-1_all.deb ... 2024-09-14T06:49:26.003 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-14T06:49:26.018 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-14T06:49:26.029 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../045-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-14T06:49:26.069 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-14T06:49:26.254 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-14T06:49:26.280 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../039-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-14T06:49:26.320 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-14T06:49:26.381 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-14T06:49:26.407 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../046-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-14T06:49:26.447 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-14T06:49:26.632 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-14T06:49:26.658 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../040-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-14T06:49:26.698 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-14T06:49:26.750 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-webob. 2024-09-14T06:49:26.776 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../047-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-14T06:49:26.816 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-14T06:49:27.001 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-natsort. 2024-09-14T06:49:27.017 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../041-python3-natsort_8.0.2-1_all.deb ... 2024-09-14T06:49:27.052 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-14T06:49:27.178 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-waitress. 2024-09-14T06:49:27.204 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../048-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-14T06:49:27.248 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-14T06:49:27.396 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-logutils. 2024-09-14T06:49:27.422 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../042-python3-logutils_0.3.3-8_all.deb ... 2024-09-14T06:49:27.463 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-14T06:49:27.791 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-14T06:49:27.798 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-tempita. 2024-09-14T06:49:27.817 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../043-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-14T06:49:27.824 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../049-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-14T06:49:27.857 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-14T06:49:27.864 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-14T06:49:28.168 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-paste. 2024-09-14T06:49:28.168 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-mako. 2024-09-14T06:49:28.194 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../050-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-14T06:49:28.195 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../044-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-14T06:49:28.234 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-14T06:49:28.235 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-14T06:49:28.529 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-14T06:49:28.544 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../051-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-14T06:49:28.563 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-14T06:49:28.587 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-14T06:49:28.590 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../045-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-14T06:49:28.630 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-14T06:49:28.874 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-14T06:49:28.882 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-14T06:49:28.889 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../046-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-14T06:49:28.908 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../052-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-14T06:49:28.924 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-14T06:49:28.948 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-14T06:49:29.244 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-webob. 2024-09-14T06:49:29.251 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-webtest. 2024-09-14T06:49:29.270 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../047-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-14T06:49:29.278 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../053-python3-webtest_2.0.35-1_all.deb ... 2024-09-14T06:49:29.310 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-14T06:49:29.318 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-14T06:49:29.629 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pecan. 2024-09-14T06:49:29.655 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-waitress. 2024-09-14T06:49:29.656 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../054-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-14T06:49:29.682 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../048-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-14T06:49:29.704 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-14T06:49:29.725 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-14T06:49:30.033 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-tempita. 2024-09-14T06:49:30.034 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-certifi. 2024-09-14T06:49:30.060 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../049-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-14T06:49:30.061 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../055-python3-certifi_2020.6.20-1_all.deb ... 2024-09-14T06:49:30.099 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-14T06:49:30.101 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-14T06:49:30.369 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-paste. 2024-09-14T06:49:30.396 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../050-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-14T06:49:30.420 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-idna. 2024-09-14T06:49:30.427 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-14T06:49:30.447 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../056-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-14T06:49:30.487 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-14T06:49:30.780 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-14T06:49:30.796 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../051-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-14T06:49:30.798 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-urllib3. 2024-09-14T06:49:30.825 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../057-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-14T06:49:30.838 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-14T06:49:30.865 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-14T06:49:31.133 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-14T06:49:31.159 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../052-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-14T06:49:31.199 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-14T06:49:31.218 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-requests. 2024-09-14T06:49:31.245 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../058-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-14T06:49:31.285 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-14T06:49:31.478 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-webtest. 2024-09-14T06:49:31.504 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../053-python3-webtest_2.0.35-1_all.deb ... 2024-09-14T06:49:31.544 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-14T06:49:31.555 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-14T06:49:31.582 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../059-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-14T06:49:31.612 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-14T06:49:31.805 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pecan. 2024-09-14T06:49:31.832 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../054-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-14T06:49:31.863 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-14T06:49:31.924 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-14T06:49:31.940 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../060-ceph-mgr-modules-core_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:31.984 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mgr-modules-core (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:32.210 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-certifi. 2024-09-14T06:49:32.237 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../055-python3-certifi_2020.6.20-1_all.deb ... 2024-09-14T06:49:32.276 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-14T06:49:32.363 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-14T06:49:32.389 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../061-libsqlite3-mod-ceph_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:32.429 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libsqlite3-mod-ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:32.613 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-idna. 2024-09-14T06:49:32.639 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../056-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-14T06:49:33.091 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-14T06:49:33.118 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../062-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-14T06:49:33.121 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-14T06:49:33.508 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-14T06:49:33.791 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-urllib3. 2024-09-14T06:49:33.818 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../057-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-14T06:49:33.849 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-14T06:49:33.885 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-openssl. 2024-09-14T06:49:33.912 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../063-python3-openssl_21.0.0-1_all.deb ... 2024-09-14T06:49:33.952 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-14T06:49:34.113 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-requests. 2024-09-14T06:49:34.126 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../058-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-14T06:49:34.161 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-14T06:49:34.289 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mgr. 2024-09-14T06:49:34.315 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../064-ceph-mgr_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:34.355 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mgr (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:34.438 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-14T06:49:34.465 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../059-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-14T06:49:34.513 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-14T06:49:34.717 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mon. 2024-09-14T06:49:34.743 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../065-ceph-mon_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:34.783 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mon (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:34.816 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-14T06:49:34.831 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../060-ceph-mgr-modules-core_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:34.866 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mgr-modules-core (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:35.254 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-14T06:49:35.281 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../061-libsqlite3-mod-ceph_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:35.321 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libsqlite3-mod-ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:35.404 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-osd. 2024-09-14T06:49:35.431 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../066-ceph-osd_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:35.474 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-osd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:35.607 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-14T06:49:35.633 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../062-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-14T06:49:35.673 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-14T06:49:35.960 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-openssl. 2024-09-14T06:49:35.987 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../063-python3-openssl_21.0.0-1_all.deb ... 2024-09-14T06:49:36.026 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-14T06:49:36.362 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mgr. 2024-09-14T06:49:36.389 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../064-ceph-mgr_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:36.437 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mgr (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:36.515 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph. 2024-09-14T06:49:36.530 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../067-ceph_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:36.573 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:36.849 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mon. 2024-09-14T06:49:36.876 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../065-ceph-mon_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:36.892 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-fuse. 2024-09-14T06:49:36.915 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mon (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:36.919 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../068-ceph-fuse_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:36.959 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:37.444 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-osd. 2024-09-14T06:49:37.471 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../066-ceph-osd_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:37.510 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-osd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:37.546 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mds. 2024-09-14T06:49:37.573 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../069-ceph-mds_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:37.604 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mds (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:37.974 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package cephadm. 2024-09-14T06:49:38.001 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../070-cephadm_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:38.040 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:38.360 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-asyncssh. 2024-09-14T06:49:38.387 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../071-python3-asyncssh_2.5.0-1_all.deb ... 2024-09-14T06:49:38.426 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-09-14T06:49:38.588 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph. 2024-09-14T06:49:38.615 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../067-ceph_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:38.655 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:38.745 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-14T06:49:38.761 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../072-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-14T06:49:38.804 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-14T06:49:39.224 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-fuse. 2024-09-14T06:49:39.251 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../068-ceph-fuse_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:39.290 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:39.610 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mds. 2024-09-14T06:49:39.626 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../069-ceph-mds_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:39.660 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mds (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:39.843 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-babel. 2024-09-14T06:49:39.870 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../073-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-14T06:49:39.909 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-14T06:49:40.046 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package cephadm. 2024-09-14T06:49:40.074 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../070-cephadm_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:40.113 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:40.237 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-jinja2. 2024-09-14T06:49:40.264 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../074-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-14T06:49:40.303 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-14T06:49:40.441 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-asyncssh. 2024-09-14T06:49:40.468 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../071-python3-asyncssh_2.5.0-1_all.deb ... 2024-09-14T06:49:40.507 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-09-14T06:49:40.673 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-14T06:49:40.700 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../075-ceph-mgr-cephadm_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:40.740 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mgr-cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:40.810 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-14T06:49:40.825 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../072-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-14T06:49:40.868 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-14T06:49:41.093 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-14T06:49:41.120 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../076-python3-repoze.lru_0.7-2_all.deb ... 2024-09-14T06:49:41.160 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-14T06:49:41.463 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-routes. 2024-09-14T06:49:41.490 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../077-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-14T06:49:41.529 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-14T06:49:41.866 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-14T06:49:41.893 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../078-ceph-mgr-dashboard_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:41.906 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-babel. 2024-09-14T06:49:41.932 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mgr-dashboard (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:41.934 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../073-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-14T06:49:41.973 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-14T06:49:42.301 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-jinja2. 2024-09-14T06:49:42.328 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../074-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-14T06:49:42.392 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-14T06:49:42.762 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-14T06:49:42.789 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../075-ceph-mgr-cephadm_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:42.829 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mgr-cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:43.165 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-14T06:49:43.193 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../076-python3-repoze.lru_0.7-2_all.deb ... 2024-09-14T06:49:43.231 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-14T06:49:43.543 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-routes. 2024-09-14T06:49:43.570 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../077-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-14T06:49:43.609 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-14T06:49:43.971 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-14T06:49:43.998 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../078-ceph-mgr-dashboard_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:44.037 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mgr-dashboard (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:44.630 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-14T06:49:44.659 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../079-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-14T06:49:44.704 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-14T06:49:45.158 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-joblib. 2024-09-14T06:49:45.185 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../080-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-14T06:49:45.216 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-14T06:49:45.579 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-14T06:49:45.606 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../081-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-14T06:49:45.645 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-14T06:49:45.948 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-sklearn. 2024-09-14T06:49:45.974 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../082-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-14T06:49:46.091 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-14T06:49:46.784 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-14T06:49:46.812 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../083-ceph-mgr-diskprediction-local_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:46.850 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mgr-diskprediction-local (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:47.009 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-14T06:49:47.023 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../079-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-14T06:49:47.067 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-14T06:49:47.587 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-joblib. 2024-09-14T06:49:47.614 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../080-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-14T06:49:47.653 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-14T06:49:47.962 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-cachetools. 2024-09-14T06:49:47.990 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../084-python3-cachetools_5.0.0-1_all.deb ... 2024-09-14T06:49:48.024 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-14T06:49:48.028 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-14T06:49:48.051 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../081-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-14T06:49:48.090 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-14T06:49:48.315 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-14T06:49:48.343 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../085-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-14T06:49:48.377 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-sklearn. 2024-09-14T06:49:48.382 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-14T06:49:48.404 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../082-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-14T06:49:48.520 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-14T06:49:48.702 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-14T06:49:48.729 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../086-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-14T06:49:48.768 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-14T06:49:49.080 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-rsa. 2024-09-14T06:49:49.107 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../087-python3-rsa_4.8-1_all.deb ... 2024-09-14T06:49:49.146 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-14T06:49:49.304 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-14T06:49:49.333 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../083-ceph-mgr-diskprediction-local_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:49.442 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-google-auth. 2024-09-14T06:49:49.469 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../088-python3-google-auth_1.5.1-3_all.deb ... 2024-09-14T06:49:49.504 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mgr-diskprediction-local (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:49.508 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-14T06:49:49.787 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-14T06:49:49.802 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../089-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-14T06:49:49.836 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-14T06:49:50.148 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-websocket. 2024-09-14T06:49:50.176 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../090-python3-websocket_1.2.3-1_all.deb ... 2024-09-14T06:49:50.214 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-14T06:49:50.474 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-cachetools. 2024-09-14T06:49:50.501 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../084-python3-cachetools_5.0.0-1_all.deb ... 2024-09-14T06:49:50.535 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-14T06:49:50.540 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-14T06:49:50.562 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../091-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-14T06:49:50.601 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-14T06:49:50.826 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-14T06:49:51.075 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../085-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-14T06:49:51.075 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-14T06:49:51.155 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-14T06:49:51.182 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../086-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-14T06:49:51.213 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-14T06:49:51.257 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-14T06:49:51.273 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../092-ceph-mgr-k8sevents_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:51.315 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-mgr-k8sevents (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:51.491 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-rsa. 2024-09-14T06:49:51.518 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../087-python3-rsa_4.8-1_all.deb ... 2024-09-14T06:49:51.557 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-14T06:49:51.584 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-14T06:49:51.612 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../093-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-14T06:49:51.651 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-14T06:49:51.870 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-google-auth. 2024-09-14T06:49:51.897 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../088-python3-google-auth_1.5.1-3_all.deb ... 2024-09-14T06:49:51.936 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-14T06:49:52.004 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-14T06:49:52.032 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../094-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-14T06:49:52.070 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-14T06:49:52.223 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-14T06:49:52.250 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../089-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-14T06:49:52.289 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-14T06:49:52.332 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package jq. 2024-09-14T06:49:52.360 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../095-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-14T06:49:52.398 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-14T06:49:52.634 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-websocket. 2024-09-14T06:49:52.643 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package socat. 2024-09-14T06:49:52.659 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../096-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-14T06:49:52.661 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../090-python3-websocket_1.2.3-1_all.deb ... 2024-09-14T06:49:52.693 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-14T06:49:52.700 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-14T06:49:52.996 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package xmlstarlet. 2024-09-14T06:49:53.024 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../097-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-14T06:49:53.029 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-14T06:49:53.056 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../091-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-14T06:49:53.062 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-14T06:49:53.095 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-14T06:49:53.400 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-test. 2024-09-14T06:49:53.425 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../098-ceph-test_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:53.459 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-test (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:53.784 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-14T06:49:53.800 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../092-ceph-mgr-k8sevents_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:53.842 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-mgr-k8sevents (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:54.287 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-14T06:49:54.314 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../093-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-14T06:49:54.353 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-14T06:49:54.640 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-14T06:49:54.667 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../094-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-14T06:49:54.697 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-14T06:49:54.950 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package jq. 2024-09-14T06:49:54.978 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../095-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-14T06:49:55.017 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-14T06:49:55.255 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package socat. 2024-09-14T06:49:55.271 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../096-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-14T06:49:55.313 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-14T06:49:55.608 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package xmlstarlet. 2024-09-14T06:49:55.624 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../097-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-14T06:49:55.679 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-14T06:49:55.988 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package ceph-volume. 2024-09-14T06:49:56.016 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../099-ceph-volume_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:56.028 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-test. 2024-09-14T06:49:56.045 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../098-ceph-test_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:56.054 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking ceph-volume (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:56.087 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-test (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:56.350 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-14T06:49:56.367 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../100-libcephfs-dev_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:56.409 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking libcephfs-dev (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:56.720 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package lua-socket:amd64. 2024-09-14T06:49:56.748 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../101-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2024-09-14T06:49:56.786 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-14T06:49:57.116 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package lua-sec:amd64. 2024-09-14T06:49:57.144 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../102-lua-sec_1.0.2-1_amd64.deb ... 2024-09-14T06:49:57.182 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2024-09-14T06:49:57.553 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package nvme-cli. 2024-09-14T06:49:57.582 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../103-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-14T06:49:57.620 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-14T06:49:58.034 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package pkg-config. 2024-09-14T06:49:58.062 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../104-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2024-09-14T06:49:58.100 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2024-09-14T06:49:58.345 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-09-14T06:49:58.361 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../105-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-09-14T06:49:58.403 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-09-14T06:49:58.607 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package ceph-volume. 2024-09-14T06:49:58.635 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../099-ceph-volume_19.3.0-4775-g7f24e589-1jammy_all.deb ... 2024-09-14T06:49:58.674 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking ceph-volume (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:58.781 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-14T06:49:58.810 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../106-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-14T06:49:58.847 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-14T06:49:58.978 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-14T06:49:59.006 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../100-libcephfs-dev_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:49:59.044 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking libcephfs-dev (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:49:59.134 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pastescript. 2024-09-14T06:49:59.162 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../107-python3-pastescript_2.0.2-4_all.deb ... 2024-09-14T06:49:59.200 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-14T06:49:59.347 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package lua-socket:amd64. 2024-09-14T06:49:59.375 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../101-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2024-09-14T06:49:59.413 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-14T06:49:59.529 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pluggy. 2024-09-14T06:49:59.557 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../108-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-14T06:49:59.595 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-14T06:49:59.718 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package lua-sec:amd64. 2024-09-14T06:49:59.746 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../102-lua-sec_1.0.2-1_amd64.deb ... 2024-09-14T06:49:59.784 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2024-09-14T06:49:59.924 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-psutil. 2024-09-14T06:49:59.952 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../109-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-14T06:49:59.990 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-14T06:50:00.147 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package nvme-cli. 2024-09-14T06:50:00.174 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../103-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-14T06:50:00.213 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-14T06:50:00.320 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-py. 2024-09-14T06:50:00.346 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../110-python3-py_1.10.0-1_all.deb ... 2024-09-14T06:50:00.392 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-14T06:50:00.635 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package pkg-config. 2024-09-14T06:50:00.662 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../104-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2024-09-14T06:50:00.701 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2024-09-14T06:50:00.746 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pygments. 2024-09-14T06:50:00.775 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../111-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-14T06:50:00.812 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-14T06:50:00.963 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-09-14T06:50:00.979 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../105-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-09-14T06:50:01.024 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-09-14T06:50:01.236 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-14T06:50:01.264 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../112-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-14T06:50:01.293 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-14T06:50:01.457 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-14T06:50:01.485 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../106-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-14T06:50:01.523 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-14T06:50:01.604 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-toml. 2024-09-14T06:50:01.633 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../113-python3-toml_0.10.2-1_all.deb ... 2024-09-14T06:50:01.671 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-14T06:50:01.834 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pastescript. 2024-09-14T06:50:01.862 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../107-python3-pastescript_2.0.2-4_all.deb ... 2024-09-14T06:50:01.900 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-14T06:50:01.974 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-pytest. 2024-09-14T06:50:02.002 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../114-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-14T06:50:02.040 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-14T06:50:02.237 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pluggy. 2024-09-14T06:50:02.268 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../108-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-14T06:50:02.304 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-14T06:50:02.352 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package python3-simplejson. 2024-09-14T06:50:02.369 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../115-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-14T06:50:02.410 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-14T06:50:02.640 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-psutil. 2024-09-14T06:50:02.668 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../109-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-14T06:50:02.679 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-14T06:50:02.696 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../116-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-14T06:50:02.706 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-14T06:50:02.737 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-14T06:50:03.051 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-py. 2024-09-14T06:50:03.079 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../110-python3-py_1.10.0-1_all.deb ... 2024-09-14T06:50:03.126 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-14T06:50:03.425 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package radosgw. 2024-09-14T06:50:03.446 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pygments. 2024-09-14T06:50:03.453 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../117-radosgw_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:50:03.474 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../111-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-14T06:50:03.491 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking radosgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:03.512 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-14T06:50:03.975 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-14T06:50:04.003 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../112-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-14T06:50:04.041 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-14T06:50:04.360 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-toml. 2024-09-14T06:50:04.361 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package rbd-fuse. 2024-09-14T06:50:04.388 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../113-python3-toml_0.10.2-1_all.deb ... 2024-09-14T06:50:04.389 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../118-rbd-fuse_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:50:04.427 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-14T06:50:04.427 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking rbd-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:04.755 INFO:teuthology.orchestra.run.smithi169.stdout:Selecting previously unselected package rbd-mirror. 2024-09-14T06:50:04.771 INFO:teuthology.orchestra.run.smithi169.stdout:Preparing to unpack .../119-rbd-mirror_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:50:04.813 INFO:teuthology.orchestra.run.smithi169.stdout:Unpacking rbd-mirror (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:04.980 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-pytest. 2024-09-14T06:50:05.008 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../114-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-14T06:50:05.046 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-14T06:50:05.301 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-14T06:50:05.375 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package python3-simplejson. 2024-09-14T06:50:05.402 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../115-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-14T06:50:05.441 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-14T06:50:05.560 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-14T06:50:05.669 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-14T06:50:05.719 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-14T06:50:05.748 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../116-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-14T06:50:05.785 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-14T06:50:05.935 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-14T06:50:06.238 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-14T06:50:06.522 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package radosgw. 2024-09-14T06:50:06.551 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../117-radosgw_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:50:06.589 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking radosgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:06.637 INFO:teuthology.orchestra.run.smithi169.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-14T06:50:06.704 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:07.238 INFO:teuthology.orchestra.run.smithi169.stdout:Adding system user cephadm....done 2024-09-14T06:50:07.386 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-14T06:50:07.484 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package rbd-fuse. 2024-09-14T06:50:07.501 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../118-rbd-fuse_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:50:07.541 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking rbd-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:07.645 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-14T06:50:07.878 INFO:teuthology.orchestra.run.smithi129.stdout:Selecting previously unselected package rbd-mirror. 2024-09-14T06:50:07.906 INFO:teuthology.orchestra.run.smithi129.stdout:Preparing to unpack .../119-rbd-mirror_19.3.0-4775-g7f24e589-1jammy_amd64.deb ... 2024-09-14T06:50:07.944 INFO:teuthology.orchestra.run.smithi129.stdout:Unpacking rbd-mirror (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:07.955 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-09-14T06:50:08.105 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-14T06:50:08.348 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-14T06:50:08.433 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-14T06:50:08.600 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-14T06:50:08.691 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-14T06:50:08.726 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-14T06:50:08.817 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-14T06:50:09.019 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-14T06:50:09.080 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-14T06:50:09.371 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-14T06:50:09.431 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-14T06:50:09.639 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-14T06:50:09.858 INFO:teuthology.orchestra.run.smithi129.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-14T06:50:09.907 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2024-09-14T06:50:09.926 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:10.167 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-14T06:50:10.434 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-14T06:50:10.468 INFO:teuthology.orchestra.run.smithi129.stdout:Adding system user cephadm....done 2024-09-14T06:50:10.624 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-14T06:50:10.694 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-14T06:50:10.900 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-14T06:50:10.944 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-ceph-argparse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:11.160 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-09-14T06:50:11.204 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-14T06:50:11.286 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-14T06:50:11.330 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libnbd0 (1.10.5-1) ... 2024-09-14T06:50:11.456 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-14T06:50:11.546 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-14T06:50:11.565 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2024-09-14T06:50:11.683 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-14T06:50:11.788 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-14T06:50:11.914 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-14T06:50:11.959 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2024-09-14T06:50:12.216 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-14T06:50:12.455 INFO:teuthology.orchestra.run.smithi169.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2024-09-14T06:50:12.498 INFO:teuthology.orchestra.run.smithi169.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2024-09-14T06:50:12.562 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-14T06:50:12.601 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-14T06:50:12.705 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-14T06:50:12.860 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-14T06:50:13.064 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-14T06:50:13.129 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2024-09-14T06:50:13.190 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-14T06:50:13.389 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-14T06:50:13.450 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-14T06:50:13.631 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-14T06:50:13.717 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-14T06:50:13.882 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-14T06:50:14.002 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up zip (3.0-12build2) ... 2024-09-14T06:50:14.128 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-14T06:50:14.141 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-ceph-argparse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:14.468 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-14T06:50:14.594 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libnbd0 (1.10.5-1) ... 2024-09-14T06:50:14.719 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-14T06:50:14.813 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-14T06:50:14.845 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2024-09-14T06:50:14.971 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-14T06:50:15.072 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-14T06:50:15.182 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-14T06:50:15.231 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2024-09-14T06:50:15.308 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-14T06:50:15.580 INFO:teuthology.orchestra.run.smithi129.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2024-09-14T06:50:15.609 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-09-14T06:50:15.613 INFO:teuthology.orchestra.run.smithi129.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2024-09-14T06:50:15.692 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-14T06:50:15.818 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-14T06:50:15.977 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-14T06:50:16.186 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-14T06:50:16.228 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-14T06:50:16.295 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-14T06:50:16.546 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-14T06:50:16.629 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-14T06:50:16.798 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-14T06:50:16.955 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-14T06:50:17.124 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up zip (3.0-12build2) ... 2024-09-14T06:50:17.233 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-14T06:50:17.298 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-14T06:50:17.541 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-14T06:50:17.659 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:17.885 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-14T06:50:17.968 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-14T06:50:18.136 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-14T06:50:18.287 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-14T06:50:18.303 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-14T06:50:18.413 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-14T06:50:18.697 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-09-14T06:50:19.082 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-14T06:50:19.316 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-14T06:50:19.501 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2024-09-14T06:50:19.684 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-14T06:50:19.935 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:50:19.977 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-14T06:50:20.072 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-14T06:50:20.361 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-14T06:50:20.488 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-14T06:50:20.537 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-14T06:50:20.615 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-14T06:50:20.780 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-14T06:50:20.905 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:20.916 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-14T06:50:21.175 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-14T06:50:21.198 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-14T06:50:21.410 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:50:21.516 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-14T06:50:21.535 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-14T06:50:21.854 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-14T06:50:22.113 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-14T06:50:22.381 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-14T06:50:22.623 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-14T06:50:22.663 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2024-09-14T06:50:22.866 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-14T06:50:22.915 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:50:23.049 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-14T06:50:23.142 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-14T06:50:23.268 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-14T06:50:23.308 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-14T06:50:23.435 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-14T06:50:23.560 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-14T06:50:23.561 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-14T06:50:23.687 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-14T06:50:23.829 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-14T06:50:23.997 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-14T06:50:24.080 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-14T06:50:24.297 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-14T06:50:24.332 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:50:24.457 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-14T06:50:24.532 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-14T06:50:24.642 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2024-09-14T06:50:24.758 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-14T06:50:24.768 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-14T06:50:24.857 INFO:teuthology.orchestra.run.smithi169.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-14T06:50:25.017 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-14T06:50:25.095 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-14T06:50:25.220 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-14T06:50:25.277 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-14T06:50:25.528 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-14T06:50:25.605 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-14T06:50:25.754 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-14T06:50:25.864 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up lua-any (27ubuntu1) ... 2024-09-14T06:50:25.965 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-14T06:50:26.005 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-14T06:50:26.131 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-14T06:50:26.225 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:50:26.351 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-14T06:50:26.407 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-14T06:50:26.534 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-14T06:50:26.685 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-14T06:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-14T06:50:26.986 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-14T06:50:27.118 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-14T06:50:27.262 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-14T06:50:27.369 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-14T06:50:27.388 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-14T06:50:27.479 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2024-09-14T06:50:27.605 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-14T06:50:27.656 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-14T06:50:27.694 INFO:teuthology.orchestra.run.smithi129.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-14T06:50:27.973 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-14T06:50:28.018 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-14T06:50:28.099 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-14T06:50:28.277 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-14T06:50:28.484 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-14T06:50:28.665 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-14T06:50:28.743 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up lua-any (27ubuntu1) ... 2024-09-14T06:50:28.869 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-14T06:50:28.955 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-14T06:50:29.112 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-14T06:50:29.221 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-14T06:50:29.308 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-14T06:50:29.434 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up librados2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:29.547 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-14T06:50:29.560 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libsqlite3-mod-ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:29.686 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2024-09-14T06:50:29.848 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-14T06:50:30.022 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libcephfs2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:30.116 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-14T06:50:30.148 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libradosstriper1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:30.242 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-14T06:50:30.266 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-14T06:50:30.518 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-14T06:50:30.526 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up librbd1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:30.652 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mgr-modules-core (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:30.778 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:30.863 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-14T06:50:30.993 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-14T06:50:30.993 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-14T06:50:31.122 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-14T06:50:31.454 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up libcephfs-dev (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:31.499 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-14T06:50:31.580 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-rados (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:31.697 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up librgw2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:31.784 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-14T06:50:31.798 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-14T06:50:32.136 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-14T06:50:32.262 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up librados2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:32.397 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libsqlite3-mod-ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:32.523 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2024-09-14T06:50:32.859 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libcephfs2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:32.952 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-rbd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:32.985 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libradosstriper1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.078 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up rbd-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.111 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-14T06:50:33.204 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-rgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.322 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up python3-cephfs (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.362 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up librbd1 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.439 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.472 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mgr-modules-core (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.598 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:33.814 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-14T06:50:33.814 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-14T06:50:33.977 INFO:teuthology.orchestra.run.smithi169.stdout:Adding group ceph....done 2024-09-14T06:50:34.282 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up libcephfs-dev (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:34.317 INFO:teuthology.orchestra.run.smithi169.stdout:Adding system user ceph....done 2024-09-14T06:50:34.425 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-rados (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:34.477 INFO:teuthology.orchestra.run.smithi169.stdout:Setting system user ceph properties....done 2024-09-14T06:50:34.497 INFO:teuthology.orchestra.run.smithi169.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-14T06:50:34.551 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up librgw2 (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:34.640 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-14T06:50:34.652 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-14T06:50:34.957 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-14T06:50:35.415 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-test (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:35.541 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up radosgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:35.839 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-rbd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:35.965 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up rbd-fuse (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:36.000 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-14T06:50:36.000 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-14T06:50:36.091 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-rgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:36.267 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up python3-cephfs (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:36.393 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-common (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:36.443 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-base (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:36.794 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-14T06:50:36.894 INFO:teuthology.orchestra.run.smithi129.stdout:Adding group ceph....done 2024-09-14T06:50:37.262 INFO:teuthology.orchestra.run.smithi129.stdout:Adding system user ceph....done 2024-09-14T06:50:37.278 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mds (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:37.439 INFO:teuthology.orchestra.run.smithi129.stdout:Setting system user ceph properties....done 2024-09-14T06:50:37.452 INFO:teuthology.orchestra.run.smithi129.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-14T06:50:37.511 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-14T06:50:37.511 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-14T06:50:37.586 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-14T06:50:37.900 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-14T06:50:37.938 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up rbd-mirror (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:38.147 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T06:50:38.147 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T06:50:38.377 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-test (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:38.503 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up radosgw (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:38.615 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mgr (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:38.831 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-14T06:50:38.831 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-14T06:50:38.988 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-14T06:50:38.988 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-14T06:50:39.413 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-base (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:39.468 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-osd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:39.764 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-14T06:50:39.764 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-14T06:50:39.768 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-14T06:50:40.280 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mgr-k8sevents (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:40.315 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mds (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:40.524 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mgr-diskprediction-local (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:40.541 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-14T06:50:40.541 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-14T06:50:40.726 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mon (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.006 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-14T06:50:41.006 INFO:teuthology.orchestra.run.smithi169.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-14T06:50:41.016 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up rbd-mirror (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.226 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T06:50:41.226 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T06:50:41.465 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mgr-cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.592 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.717 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-mgr-dashboard (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.718 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mgr (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.869 INFO:teuthology.orchestra.run.smithi169.stdout:Setting up ceph-volume (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:41.937 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-14T06:50:41.937 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-14T06:50:42.221 INFO:teuthology.orchestra.run.smithi169.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2024-09-14T06:50:42.330 INFO:teuthology.orchestra.run.smithi169.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-14T06:50:42.419 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-osd (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:42.483 INFO:teuthology.orchestra.run.smithi169.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-14T06:50:42.677 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-14T06:50:42.677 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-14T06:50:43.154 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mgr-k8sevents (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:43.249 INFO:teuthology.orchestra.run.smithi169.stdout:Processing triggers for install-info (6.8-4build1) ... 2024-09-14T06:50:43.281 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mgr-diskprediction-local (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:43.431 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mon (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:43.644 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-14T06:50:43.644 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-14T06:50:43.848 INFO:teuthology.orchestra.run.smithi169.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-14T06:50:43.854 DEBUG:teuthology.parallel:result is None 2024-09-14T06:50:44.124 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mgr-cephadm (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:44.275 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:44.401 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-mgr-dashboard (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:44.527 INFO:teuthology.orchestra.run.smithi129.stdout:Setting up ceph-volume (19.3.0-4775-g7f24e589-1jammy) ... 2024-09-14T06:50:44.880 INFO:teuthology.orchestra.run.smithi129.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2024-09-14T06:50:44.989 INFO:teuthology.orchestra.run.smithi129.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-14T06:50:45.150 INFO:teuthology.orchestra.run.smithi129.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-14T06:50:46.411 INFO:teuthology.orchestra.run.smithi129.stdout:Processing triggers for install-info (6.8-4build1) ... 2024-09-14T06:50:47.056 INFO:teuthology.orchestra.run.smithi129.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-14T06:50:47.061 DEBUG:teuthology.parallel:result is None 2024-09-14T06:50:47.061 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T06:50:47.209 DEBUG:teuthology.orchestra.run.smithi129:> dpkg-query -W -f '${Version}' ceph 2024-09-14T06:50:47.227 INFO:teuthology.orchestra.run.smithi129.stdout:19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:50:47.228 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:50:47.228 INFO:teuthology.task.install:The correct ceph version 19.3.0-4775-g7f24e589-1jammy is installed. 2024-09-14T06:50:47.230 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=7f24e589328ae9e639c0009d93340e5b235d0a47 2024-09-14T06:50:47.365 DEBUG:teuthology.orchestra.run.smithi169:> dpkg-query -W -f '${Version}' ceph 2024-09-14T06:50:47.384 INFO:teuthology.orchestra.run.smithi169.stdout:19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:50:47.385 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4775-g7f24e589-1jammy 2024-09-14T06:50:47.385 INFO:teuthology.task.install:The correct ceph version 19.3.0-4775-g7f24e589-1jammy is installed. 2024-09-14T06:50:47.386 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T06:50:47.387 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:47.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T06:50:47.402 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:50:47.402 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T06:50:47.440 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T06:50:47.441 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:47.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T06:50:47.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T06:50:47.513 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:50:47.513 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T06:50:47.526 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T06:50:47.582 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T06:50:47.583 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:47.583 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T06:50:47.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T06:50:47.652 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:50:47.652 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T06:50:47.668 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T06:50:47.723 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T06:50:47.723 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:47.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T06:50:47.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T06:50:47.792 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:50:47.792 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T06:50:47.804 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T06:50:47.859 INFO:teuthology.run_tasks:Running task ceph... 2024-09-14T06:50:47.946 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-14T06:50:47.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 777 /var/log/ceph 2024-09-14T06:50:47.949 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 777 /var/log/ceph 2024-09-14T06:50:47.961 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-14T06:50:47.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-14T06:50:48.005 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-14T06:50:48.017 INFO:tasks.ceph:Creating extra log directories... 2024-09-14T06:50:48.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-14T06:50:48.058 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-14T06:50:48.084 INFO:tasks.ceph:Creating ceph cluster cluster1... 2024-09-14T06:50:48.085 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-14T06:50:48.085 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398', 'branch': 'main', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7900398', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 950, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 5839, 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47', 'sleep_before_teardown': 0, 'subset': '99/128', 'suite': 'rbd', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a', 'targets': {'smithi129.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCewASal6XC5348xuSGBkugXPsEDpkJbeCgS3/Cs+587P8hWBK2W/EOtSErdQ4R+RR1SNAalWl4DZy/nL/NQoSw=', 'smithi169.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSdrVcJWaQlujfB2Sjg4WwVO1/3Ud6hUEmt60A7ZtRUWilyOrIDw3caPOYXMYTtwBVS3Qu54tSO7pau77hUBbo='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-10_20:16:17', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-14T06:50:48.085 DEBUG:teuthology.orchestra.run.smithi129:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-14T06:50:48.113 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-14T06:50:48.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T06:50:48.162 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T06:50:48.175 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:48.175 DEBUG:teuthology.orchestra.run.smithi129:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T06:50:48.221 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T06:50:48.221 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_1 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 880 Links: 1 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 06:45:20.200337234 +0000 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 06:45:19.936339593 +0000 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 06:45:19.936339593 +0000 2024-09-14T06:50:48.273 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: - 2024-09-14T06:50:48.274 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T06:50:48.327 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T06:50:48.327 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T06:50:48.327 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000443253 s, 1.2 MB/s 2024-09-14T06:50:48.329 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T06:50:48.378 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_2 2024-09-14T06:50:48.425 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T06:50:48.425 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:50:48.426 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 895 Links: 1 2024-09-14T06:50:48.426 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:50:48.426 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 06:45:20.696332804 +0000 2024-09-14T06:50:48.426 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 06:45:20.456334948 +0000 2024-09-14T06:50:48.426 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 06:45:20.456334948 +0000 2024-09-14T06:50:48.426 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: - 2024-09-14T06:50:48.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T06:50:48.483 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T06:50:48.483 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T06:50:48.483 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.00042181 s, 1.2 MB/s 2024-09-14T06:50:48.485 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T06:50:48.534 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_3 2024-09-14T06:50:48.584 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T06:50:48.584 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:50:48.585 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 918 Links: 1 2024-09-14T06:50:48.585 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:50:48.585 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 06:45:21.156328695 +0000 2024-09-14T06:50:48.585 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 06:45:20.936330660 +0000 2024-09-14T06:50:48.585 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 06:45:20.936330660 +0000 2024-09-14T06:50:48.585 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: - 2024-09-14T06:50:48.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T06:50:48.639 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T06:50:48.640 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T06:50:48.640 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000442529 s, 1.2 MB/s 2024-09-14T06:50:48.641 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T06:50:48.694 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_4 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 935 Links: 1 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 06:45:21.596324764 +0000 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 06:45:21.372326766 +0000 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 06:45:21.372326766 +0000 2024-09-14T06:50:48.741 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: - 2024-09-14T06:50:48.741 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T06:50:48.795 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T06:50:48.795 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T06:50:48.796 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000372896 s, 1.4 MB/s 2024-09-14T06:50:48.797 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T06:50:48.850 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:50:48.850 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi129.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T06:50:48.850 INFO:tasks.ceph:Generating config... 2024-09-14T06:50:48.851 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-14T06:50:48.851 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client] log to stderr = False 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:48.852 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:48.853 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-14T06:50:48.853 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-14T06:50:48.853 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-14T06:50:48.853 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-14T06:50:48.853 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-14T06:50:48.853 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-14T06:50:48.853 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-14T06:50:48.853 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-14T06:50:48.853 INFO:tasks.ceph:[osd] bdev async discard = True 2024-09-14T06:50:48.853 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-09-14T06:50:48.853 INFO:tasks.ceph:[osd] bluestore allocator = stupid 2024-09-14T06:50:48.853 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-14T06:50:48.853 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-14T06:50:48.853 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-14T06:50:48.854 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-14T06:50:48.854 INFO:tasks.ceph:Setting up cluster1.mon.a... 2024-09-14T06:50:48.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2024-09-14T06:50:48.928 INFO:teuthology.orchestra.run.smithi129.stdout:creating /etc/ceph/cluster1.keyring 2024-09-14T06:50:48.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2024-09-14T06:50:49.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-14T06:50:49.076 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '172.21.15.129')] 2024-09-14T06:50:49.076 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.129', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-14T06:50:49.077 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:49.077 DEBUG:teuthology.orchestra.run.smithi129:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T06:50:49.125 DEBUG:teuthology.orchestra.run.smithi129:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.129 --print /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:49.202 INFO:teuthology.orchestra.run.smithi129.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:monmaptool: generated fsid 95676e5b-ca8a-4e3d-9640-d6c7d09c88f4 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:setting min_mon_release = quincy 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:epoch 0 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:fsid 95676e5b-ca8a-4e3d-9640-d6c7d09c88f4 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:last_changed 2024-09-14T06:50:49.204956+0000 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:created 2024-09-14T06:50:49.204956+0000 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:min_mon_release 17 (quincy) 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:election_strategy: 1 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:0: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.a 2024-09-14T06:50:49.203 INFO:teuthology.orchestra.run.smithi129.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2024-09-14T06:50:49.205 DEBUG:teuthology.orchestra.run.smithi129:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T06:50:49.253 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 95676e5b-ca8a-4e3d-9640-d6c7d09c88f4... 2024-09-14T06:50:49.254 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-14T06:50:49.297 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout:[global] 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: chdir = "" 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: auth supported = cephx 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: filestore xattr use omap = true 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: mon clock drift allowed = 1.000 2024-09-14T06:50:49.326 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: osd crush chooseleaf type = 0 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: auth debug = true 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: ms die on old message = true 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: ms die on bug = true 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: mon pg warn max object skew = 0 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: osd pool default size = 2 2024-09-14T06:50:49.327 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd allow primary affinity = true 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd allow pg remap = true 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on legacy crush tunables = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on crush straw calc version zero = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on no sortbitwise = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on osd down out interval zero = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on too few osds = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: mon_allow_pool_size_one = true 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T06:50:49.328 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: osd default data pool replay window = 5 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: mon allow pool delete = true 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: mon cluster log file level = debug 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: debug asserts on shutdown = true 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: mon health detail to clog = false 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: mon host = 172.21.15.129 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: mon client directed command retry = 5 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on pool no app = False 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: ms inject socket failures = 5000 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: fsid = 95676e5b-ca8a-4e3d-9640-d6c7d09c88f4 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.329 INFO:teuthology.orchestra.run.smithi129.stdout:[osd] 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd journal size = 100 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd scrub load threshold = 5.0 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd scrub max interval = 600 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd mclock profile = high_recovery_ops 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd recover clone overlap = true 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd recovery max chunk = 1048576 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug shutdown = true 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug op order = true 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug verify stray on activate = true 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug trim objects = true 2024-09-14T06:50:49.330 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: osd open classes on start = true 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug pg log writeout = true 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: osd deep scrub update digest min age = 30 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: osd map max advance = 10 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: journal zero on create = true 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: filestore ondisk finisher threads = 3 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: filestore apply finisher threads = 3 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: bdev debug aio = true 2024-09-14T06:50:49.331 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug misdirected ops = true 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: bdev async discard = True 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: bdev enable discard = True 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: bluestore allocator = stupid 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: bluestore block size = 96636764160 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: bluestore fsck on mount = True 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: debug bluefs = 1/20 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: debug bluestore = 1/20 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: debug ms = 1 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: debug osd = 20 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: debug rocksdb = 4/10 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd full ratio = 0.9 2024-09-14T06:50:49.332 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd nearfull ratio = 0.8 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: osd failsafe full ratio = 0.95 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: osd objectstore = bluestore 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout:[mgr] 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: debug ms = 1 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: debug mgr = 20 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: debug mon = 20 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: debug auth = 20 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: mgr/telemetry/nag = false 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout:[mon] 2024-09-14T06:50:49.333 INFO:teuthology.orchestra.run.smithi129.stdout: debug ms = 1 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: debug mon = 20 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: debug paxos = 20 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: debug auth = 20 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: mon data avail warn = 5 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: mon mgr mkfs grace = 240 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd reporter subtree level = osd 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd prime pg temp = true 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T06:50:49.334 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: # 1m isn't quite enough 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: mon_down_mkfs_grace = 2m 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_filestore_osds = false 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout:[client] 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: rgw cache enabled = true 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: rgw enable ops log = true 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: rgw enable usage log = true 2024-09-14T06:50:49.335 INFO:teuthology.orchestra.run.smithi129.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: debug rbd = 20 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: debug rbd_mirror = 15 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: log to stderr = False 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: rbd default features = 125 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: rbd mirror image policy type = simple 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout:[mon.a] 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.0] 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.1] 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.336 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.2] 2024-09-14T06:50:49.337 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.337 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.337 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.3] 2024-09-14T06:50:49.337 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.337 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.338 INFO:teuthology.orchestra.run.smithi169.stdout:[global] 2024-09-14T06:50:49.338 INFO:teuthology.orchestra.run.smithi169.stdout: chdir = "" 2024-09-14T06:50:49.338 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: auth supported = cephx 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: filestore xattr use omap = true 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: mon clock drift allowed = 1.000 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: osd crush chooseleaf type = 0 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: auth debug = true 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: ms die on old message = true 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: ms die on bug = true 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.339 INFO:teuthology.orchestra.run.smithi169.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon pg warn max object skew = 0 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: osd pool default size = 2 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd allow primary affinity = true 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd allow pg remap = true 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on legacy crush tunables = false 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on crush straw calc version zero = false 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on no sortbitwise = false 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on osd down out interval zero = false 2024-09-14T06:50:49.340 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on too few osds = false 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon_allow_pool_size_one = true 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: osd default data pool replay window = 5 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon allow pool delete = true 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon cluster log file level = debug 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: debug asserts on shutdown = true 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon health detail to clog = false 2024-09-14T06:50:49.341 INFO:teuthology.orchestra.run.smithi169.stdout: mon host = 172.21.15.129 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: mon client directed command retry = 5 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on pool no app = False 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: ms inject socket failures = 5000 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: fsid = 95676e5b-ca8a-4e3d-9640-d6c7d09c88f4 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout:[osd] 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: osd journal size = 100 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: osd scrub load threshold = 5.0 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: osd scrub max interval = 600 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: osd mclock profile = high_recovery_ops 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.342 INFO:teuthology.orchestra.run.smithi169.stdout: osd recover clone overlap = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd recovery max chunk = 1048576 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug shutdown = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug op order = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug verify stray on activate = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug trim objects = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd open classes on start = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug pg log writeout = true 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd deep scrub update digest min age = 30 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.343 INFO:teuthology.orchestra.run.smithi169.stdout: osd map max advance = 10 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: journal zero on create = true 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: filestore ondisk finisher threads = 3 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: filestore apply finisher threads = 3 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: bdev debug aio = true 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug misdirected ops = true 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: bdev async discard = True 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: bdev enable discard = True 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: bluestore allocator = stupid 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: bluestore block size = 96636764160 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: bluestore fsck on mount = True 2024-09-14T06:50:49.344 INFO:teuthology.orchestra.run.smithi169.stdout: debug bluefs = 1/20 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: debug bluestore = 1/20 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: debug osd = 20 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: debug rocksdb = 4/10 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd full ratio = 0.9 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd nearfull ratio = 0.8 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: osd failsafe full ratio = 0.95 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: osd objectstore = bluestore 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout:[mgr] 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2024-09-14T06:50:49.345 INFO:teuthology.orchestra.run.smithi169.stdout: debug mgr = 20 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: debug mon = 20 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: debug auth = 20 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: mgr/telemetry/nag = false 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout:[mon] 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: debug mon = 20 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: debug paxos = 20 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: debug auth = 20 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: mon data avail warn = 5 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: mon mgr mkfs grace = 240 2024-09-14T06:50:49.346 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd reporter subtree level = osd 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd prime pg temp = true 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.347 INFO:teuthology.orchestra.run.smithi169.stdout: # 1m isn't quite enough 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: mon_down_mkfs_grace = 2m 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_filestore_osds = false 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout:[client] 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: rgw cache enabled = true 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: rgw enable ops log = true 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: rgw enable usage log = true 2024-09-14T06:50:49.348 INFO:teuthology.orchestra.run.smithi169.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: debug rbd = 20 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: debug rbd_mirror = 15 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: log to stderr = False 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: rbd default features = 125 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: rbd mirror image policy type = simple 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout:[mon.a] 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.0] 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.349 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.1] 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.2] 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.3] 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:50:49.350 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:50:49.350 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2024-09-14T06:50:49.350 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2024-09-14T06:50:49.428 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-14T06:50:49.428 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:49.428 DEBUG:teuthology.orchestra.run.smithi129:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2024-09-14T06:50:49.433 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:49.433 DEBUG:teuthology.orchestra.run.smithi129:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2024-09-14T06:50:49.481 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T06:50:49.481 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:49.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-14T06:50:49.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-14T06:50:49.545 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:50:49.545 DEBUG:teuthology.orchestra.run.smithi129:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:49.593 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:50:49.593 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:50:49.593 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-14T06:50:49.593 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-14T06:50:49.616 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:50:49.616 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:49.668 INFO:tasks.ceph:Setting up mon nodes... 2024-09-14T06:50:49.668 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-14T06:50:49.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-14T06:50:49.713 INFO:teuthology.orchestra.run.smithi129.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-14T06:50:49.716 INFO:tasks.ceph:Setting up mds nodes... 2024-09-14T06:50:49.716 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-14T06:50:49.716 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2024-09-14T06:50:49.747 INFO:teuthology.orchestra.run.smithi129.stdout:creating /etc/ceph/cluster1.client.0.keyring 2024-09-14T06:50:49.757 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2024-09-14T06:50:49.792 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2024-09-14T06:50:49.815 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-14T06:50:49.898 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-14T06:50:49.909 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-14T06:50:49.984 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-14T06:50:49.995 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-14T06:50:50.066 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-14T06:50:50.076 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-14T06:50:50.151 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-14T06:50:50.161 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-14T06:50:50.233 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-14T06:50:50.242 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-14T06:50:50.313 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-14T06:50:50.322 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-14T06:50:50.390 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-14T06:50:50.402 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-14T06:50:50.402 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi129.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T06:50:50.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2024-09-14T06:50:50.417 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:50:50.417 INFO:tasks.ceph:role: cluster1.osd.0 2024-09-14T06:50:50.417 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T06:50:50.417 DEBUG:teuthology.orchestra.run.smithi129:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-14T06:50:50.480 INFO:teuthology.orchestra.run.smithi129.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout: = sunit=0 swidth=0 blks 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T06:50:50.481 INFO:teuthology.orchestra.run.smithi129.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T06:50:50.938 INFO:teuthology.orchestra.run.smithi129.stdout:Discarding blocks...Done. 2024-09-14T06:50:50.940 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi129.front.sepia.ceph.com -o noatime 2024-09-14T06:50:50.941 DEBUG:teuthology.orchestra.run.smithi129:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2024-09-14T06:50:51.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2024-09-14T06:50:51.080 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: /sbin/restorecon: command not found 2024-09-14T06:50:51.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:50:51.081 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2024-09-14T06:50:51.136 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:50:51.136 INFO:tasks.ceph:role: cluster1.osd.1 2024-09-14T06:50:51.136 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T06:50:51.136 DEBUG:teuthology.orchestra.run.smithi129:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-14T06:50:51.199 INFO:teuthology.orchestra.run.smithi129.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T06:50:51.199 INFO:teuthology.orchestra.run.smithi129.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T06:50:51.199 INFO:teuthology.orchestra.run.smithi129.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T06:50:51.199 INFO:teuthology.orchestra.run.smithi129.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-14T06:50:51.200 INFO:teuthology.orchestra.run.smithi129.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T06:50:51.200 INFO:teuthology.orchestra.run.smithi129.stdout: = sunit=0 swidth=0 blks 2024-09-14T06:50:51.200 INFO:teuthology.orchestra.run.smithi129.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T06:50:51.200 INFO:teuthology.orchestra.run.smithi129.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-14T06:50:51.200 INFO:teuthology.orchestra.run.smithi129.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T06:50:51.200 INFO:teuthology.orchestra.run.smithi129.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T06:50:51.625 INFO:teuthology.orchestra.run.smithi129.stdout:Discarding blocks...Done. 2024-09-14T06:50:51.627 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi129.front.sepia.ceph.com -o noatime 2024-09-14T06:50:51.627 DEBUG:teuthology.orchestra.run.smithi129:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2024-09-14T06:50:51.688 DEBUG:teuthology.orchestra.run.smithi129:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2024-09-14T06:50:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:50:51.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: /sbin/restorecon: command not found 2024-09-14T06:50:51.742 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2024-09-14T06:50:51.794 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:50:51.794 INFO:tasks.ceph:role: cluster1.osd.2 2024-09-14T06:50:51.794 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T06:50:51.794 DEBUG:teuthology.orchestra.run.smithi129:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-14T06:50:51.856 INFO:teuthology.orchestra.run.smithi129.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T06:50:51.856 INFO:teuthology.orchestra.run.smithi129.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T06:50:51.856 INFO:teuthology.orchestra.run.smithi129.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T06:50:51.856 INFO:teuthology.orchestra.run.smithi129.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-14T06:50:51.857 INFO:teuthology.orchestra.run.smithi129.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T06:50:51.857 INFO:teuthology.orchestra.run.smithi129.stdout: = sunit=0 swidth=0 blks 2024-09-14T06:50:51.857 INFO:teuthology.orchestra.run.smithi129.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T06:50:51.857 INFO:teuthology.orchestra.run.smithi129.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-14T06:50:51.857 INFO:teuthology.orchestra.run.smithi129.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T06:50:51.857 INFO:teuthology.orchestra.run.smithi129.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T06:50:52.310 INFO:teuthology.orchestra.run.smithi129.stdout:Discarding blocks...Done. 2024-09-14T06:50:52.312 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi129.front.sepia.ceph.com -o noatime 2024-09-14T06:50:52.312 DEBUG:teuthology.orchestra.run.smithi129:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2024-09-14T06:50:52.420 DEBUG:teuthology.orchestra.run.smithi129:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2024-09-14T06:50:52.430 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: /sbin/restorecon: command not found 2024-09-14T06:50:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:50:52.431 DEBUG:teuthology.orchestra.run.smithi129:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:52.538 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:52.537+0000 7f01a3d907c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2024-09-14T06:50:52.542 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:52.541+0000 7f01a3d907c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2024-09-14T06:50:52.542 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:52.541+0000 7f01a3d907c0 -1 bdev(0x56088b005800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2024-09-14T06:50:52.542 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:52.541+0000 7f01a3d907c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2024-09-14T06:50:55.681 DEBUG:teuthology.orchestra.run.smithi129:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2024-09-14T06:50:55.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:55.775 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:55.773+0000 7f054d16d7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2024-09-14T06:50:55.776 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:55.773+0000 7f054d16d7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2024-09-14T06:50:55.776 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:55.773+0000 7f054d16d7c0 -1 bdev(0x5599a2cd7800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2024-09-14T06:50:55.776 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:55.773+0000 7f054d16d7c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2024-09-14T06:50:58.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2024-09-14T06:50:58.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:50:59.068 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:59.069+0000 7fbea59947c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2024-09-14T06:50:59.069 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:59.069+0000 7fbea59947c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2024-09-14T06:50:59.070 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:59.069+0000 7fbea59947c0 -1 bdev(0x561c5ffa9800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2024-09-14T06:50:59.070 INFO:teuthology.orchestra.run.smithi129.stderr:2024-09-14T06:50:59.069+0000 7fbea59947c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2024-09-14T06:51:02.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2024-09-14T06:51:02.256 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-14T06:51:02.256 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:02.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2024-09-14T06:51:02.311 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:02.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2024-09-14T06:51:02.362 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:02.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2024-09-14T06:51:02.414 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:02.414 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2024-09-14T06:51:02.467 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:02.467 DEBUG:teuthology.orchestra.run.smithi129:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2024-09-14T06:51:02.512 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.512 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2024-09-14T06:51:02.518 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.518 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2024-09-14T06:51:02.563 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.564 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2024-09-14T06:51:02.612 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.612 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2024-09-14T06:51:02.660 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.660 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2024-09-14T06:51:02.709 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.709 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2024-09-14T06:51:02.755 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.756 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2024-09-14T06:51:02.803 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:02.804 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2024-09-14T06:51:02.852 INFO:tasks.ceph:Adding keys to all mons... 2024-09-14T06:51:02.852 DEBUG:teuthology.orchestra.run.smithi129:> sudo tee -a /etc/ceph/cluster1.keyring 2024-09-14T06:51:02.864 INFO:teuthology.orchestra.run.smithi129.stdout:[mgr.x] 2024-09-14T06:51:02.864 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBJMuVmqcqkKhAAD1TMgJL+EF6ruo2XSylDag== 2024-09-14T06:51:02.864 INFO:teuthology.orchestra.run.smithi129.stdout:[osd.0] 2024-09-14T06:51:02.864 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBMMuVmvR8+IBAAocebPnEuK4Qd3kfjEJcsYQ== 2024-09-14T06:51:02.864 INFO:teuthology.orchestra.run.smithi129.stdout:[osd.1] 2024-09-14T06:51:02.864 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBPMuVmzi9SLhAAxKA3UT2Mzb/iMdtjZZEuCQ== 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout:[osd.2] 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBTMuVmZjg1BBAAA8IC9KfatPvYMy0GF7JF1g== 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout:[client.0] 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBJMuVmRKm1LBAAQJ9fq921koLxu838sLXrfg== 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror] 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBJMuVmspdYLxAAnqp+2Q9sPgKEnKSoWDGF6A== 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.0] 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBJMuVmCRmdNRAACsfW/PaTz+w2yuFQ482TeQ== 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.1] 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBJMuVmOim6OhAAJEVCXOcISRe+StLwzA02ZA== 2024-09-14T06:51:02.865 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.2] 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBKMuVmU6cTBBAAUTbyGVf5p4Rmd3EkWgOsiA== 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.3] 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBKMuVmI/AWCRAAw14wU7XMLA1o9oejUpAVGg== 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.4] 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBKMuVmkcz2DRAApfUmnciffAse27Alcgvmjw== 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.5] 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBKMuVmkGzFEhAA2D0cYi9LRBzF1trZVbn9+Q== 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.6] 2024-09-14T06:51:02.866 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBKMuVm7qJhFxAAfRxgWwoSD0OplOygjj/yRQ== 2024-09-14T06:51:02.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-14T06:51:02.953 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T06:51:02.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T06:51:03.076 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T06:51:03.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.431 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.599 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.688 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:03.865 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-14T06:51:03.865 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2024-09-14T06:51:03.877 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2024-09-14T06:51:03.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2024-09-14T06:51:03.975 DEBUG:teuthology.orchestra.run.smithi129:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2024-09-14T06:51:04.020 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2024-09-14T06:51:04.020 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2024-09-14T06:51:04.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2024-09-14T06:51:04.065 INFO:tasks.ceph.cluster1.mon.a:Started 2024-09-14T06:51:04.065 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2024-09-14T06:51:04.066 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2024-09-14T06:51:04.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2024-09-14T06:51:04.068 INFO:tasks.ceph.cluster1.mgr.x:Started 2024-09-14T06:51:04.068 DEBUG:tasks.ceph:set 0 configs 2024-09-14T06:51:04.068 INFO:tasks.ceph:Setting crush tunables to default 2024-09-14T06:51:04.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo ceph --cluster cluster1 osd crush tunables default 2024-09-14T06:51:04.278 INFO:teuthology.orchestra.run.smithi129.stderr:adjusted tunables profile to default 2024-09-14T06:51:04.295 INFO:tasks.ceph:check_enable_crimson: False 2024-09-14T06:51:04.295 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2024-09-14T06:51:04.295 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:04.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2024-09-14T06:51:04.307 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:04.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2024-09-14T06:51:04.361 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:04.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2024-09-14T06:51:04.418 DEBUG:teuthology.orchestra.run.smithi129:> sudo ceph --cluster cluster1 osd new 9c9e29ab-ca41-4253-b78f-fd771e0b1ac9 0 2024-09-14T06:51:04.662 INFO:teuthology.orchestra.run.smithi129.stdout:0 2024-09-14T06:51:04.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo ceph --cluster cluster1 osd new e2b7870c-f0b3-48f7-8c61-d69b195c1ad2 1 2024-09-14T06:51:04.885 INFO:teuthology.orchestra.run.smithi129.stdout:1 2024-09-14T06:51:04.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo ceph --cluster cluster1 osd new 8a45500a-5cc2-4dae-8608-00acc1982432 2 2024-09-14T06:51:05.095 INFO:teuthology.orchestra.run.smithi129.stdout:2 2024-09-14T06:51:05.111 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2024-09-14T06:51:05.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2024-09-14T06:51:05.114 INFO:tasks.ceph.cluster1.osd.0:Started 2024-09-14T06:51:05.114 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2024-09-14T06:51:05.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2024-09-14T06:51:05.117 INFO:tasks.ceph.cluster1.osd.1:Started 2024-09-14T06:51:05.117 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2024-09-14T06:51:05.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2024-09-14T06:51:05.120 INFO:tasks.ceph.cluster1.osd.2:Started 2024-09-14T06:51:05.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T06:51:05.301 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:05.301 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":5,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T06:51:05.096335+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:05.316 INFO:tasks.ceph.ceph_manager.cluster1:[] 2024-09-14T06:51:05.317 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-14T06:51:05.620 DEBUG:teuthology.orchestra.run.smithi129:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-14T06:51:05.800 INFO:teuthology.misc.health.smithi129.stdout: 2024-09-14T06:51:05.800 INFO:teuthology.misc.health.smithi129.stdout:{"epoch":5,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T06:51:05.096335+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:05.815 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T06:51:06.213 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T06:51:06.213+0000 7f0e055e87c0 -1 Falling back to public interface 2024-09-14T06:51:06.569 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T06:51:06.569+0000 7f99fe4767c0 -1 Falling back to public interface 2024-09-14T06:51:06.853 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:51:06.853+0000 7fea6f5f97c0 -1 Falling back to public interface 2024-09-14T06:51:07.100 INFO:tasks.ceph.cluster1.mgr.x.smithi129.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2024-09-14T06:51:07.100 INFO:tasks.ceph.cluster1.mgr.x.smithi129.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2024-09-14T06:51:07.100 INFO:tasks.ceph.cluster1.mgr.x.smithi129.stderr: from numpy import show_config as show_numpy_config 2024-09-14T06:51:10.007 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T06:51:10.005+0000 7f0e055e87c0 -1 osd.1 0 log_to_monitors true 2024-09-14T06:51:10.015 INFO:tasks.ceph.cluster1.mgr.x.smithi129.stderr:2024-09-14T06:51:10.013+0000 7f4568100640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T06:51:10.528 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T06:51:10.529+0000 7f99fe4767c0 -1 osd.2 0 log_to_monitors true 2024-09-14T06:51:10.861 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:51:10.861+0000 7fea6f5f97c0 -1 osd.0 0 log_to_monitors true 2024-09-14T06:51:12.118 DEBUG:teuthology.orchestra.run.smithi129:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-14T06:51:12.459 INFO:teuthology.misc.health.smithi129.stdout: 2024-09-14T06:51:12.459 INFO:teuthology.misc.health.smithi129.stdout:{"epoch":7,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T06:51:12.012945+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:12.479 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T06:51:13.924 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:51:13.925+0000 7fea6bdb4640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T06:51:13.944 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:51:13.945+0000 7fea633d4640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T06:51:13.966 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T06:51:13.965+0000 7f0e01591640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T06:51:13.981 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T06:51:13.981+0000 7f99fa41f640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T06:51:13.992 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T06:51:13.993+0000 7f0df93c3640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T06:51:14.004 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T06:51:14.005+0000 7f99f2251640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T06:51:18.782 DEBUG:teuthology.orchestra.run.smithi129:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-14T06:51:19.087 INFO:teuthology.misc.health.smithi129.stdout: 2024-09-14T06:51:19.087 INFO:teuthology.misc.health.smithi129.stdout:{"epoch":11,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T06:51:18.037485+0000","last_up_change":"2024-09-14T06:51:14.927617+0000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:15.054238+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6817","nonce":3974496147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6819","nonce":3974496147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6823","nonce":3974496147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6821","nonce":3974496147}]},"public_addr":"172.21.15.129:6817/3974496147","cluster_addr":"172.21.15.129:6819/3974496147","heartbeat_back_addr":"172.21.15.129:6823/3974496147","heartbeat_front_addr":"172.21.15.129:6821/3974496147","state":["exists","up"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6801","nonce":498324347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6803","nonce":498324347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6807","nonce":498324347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6805","nonce":498324347}]},"public_addr":"172.21.15.129:6801/498324347","cluster_addr":"172.21.15.129:6803/498324347","heartbeat_back_addr":"172.21.15.129:6807/498324347","heartbeat_front_addr":"172.21.15.129:6805/498324347","state":["exists","up"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6809","nonce":3388405936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6811","nonce":3388405936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6815","nonce":3388405936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6813","nonce":3388405936}]},"public_addr":"172.21.15.129:6809/3388405936","cluster_addr":"172.21.15.129:6811/3388405936","heartbeat_back_addr":"172.21.15.129:6815/3388405936","heartbeat_front_addr":"172.21.15.129:6813/3388405936","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.879825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.057602+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.575932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:19.103 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-14T06:51:19.104 INFO:tasks.ceph:Creating RBD pool 2024-09-14T06:51:19.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo ceph --cluster cluster1 osd pool create rbd 8 2024-09-14T06:51:20.091 INFO:teuthology.orchestra.run.smithi129.stderr:pool 'rbd' created 2024-09-14T06:51:20.118 DEBUG:teuthology.orchestra.run.smithi129:> rbd --cluster cluster1 pool init rbd 2024-09-14T06:51:23.134 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2024-09-14T06:51:23.134 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2024-09-14T06:51:23.135 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-14T06:51:23.450 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:23.469 INFO:teuthology.orchestra.run.smithi129.stdout:[{"version":4,"timestamp":"2024-09-14T06:51:13.984381+0000","name":"","changes":[{"name":"osd.2/osd_mclock_max_capacity_iops_ssd","new_value":"3965.878145"}]}] 2024-09-14T06:51:23.469 INFO:tasks.ceph_manager:config epoch is 4 2024-09-14T06:51:23.469 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2024-09-14T06:51:23.469 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2024-09-14T06:51:23.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2024-09-14T06:51:23.822 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:23.840 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2685020093},{"type":"v1","addr":"172.21.15.129:6825","nonce":2685020093}]},"active_addr":"172.21.15.129:6825/2685020093","active_change":"2024-09-14T06:51:09.007382+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":234361400}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":2227416573}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":1943621221}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":3539362066}]}]} 2024-09-14T06:51:23.842 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2024-09-14T06:51:23.843 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2024-09-14T06:51:23.843 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T06:51:24.151 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:24.151 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":15,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T06:51:23.104554+0000","last_up_change":"2024-09-14T06:51:14.927617+0000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:15.054238+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T06:51:19.403913+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6817","nonce":3974496147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6819","nonce":3974496147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6823","nonce":3974496147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6821","nonce":3974496147}]},"public_addr":"172.21.15.129:6817/3974496147","cluster_addr":"172.21.15.129:6819/3974496147","heartbeat_back_addr":"172.21.15.129:6823/3974496147","heartbeat_front_addr":"172.21.15.129:6821/3974496147","state":["exists","up"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6801","nonce":498324347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6803","nonce":498324347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6807","nonce":498324347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6805","nonce":498324347}]},"public_addr":"172.21.15.129:6801/498324347","cluster_addr":"172.21.15.129:6803/498324347","heartbeat_back_addr":"172.21.15.129:6807/498324347","heartbeat_front_addr":"172.21.15.129:6805/498324347","state":["exists","up"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6809","nonce":3388405936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6811","nonce":3388405936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6815","nonce":3388405936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6813","nonce":3388405936}]},"public_addr":"172.21.15.129:6809/3388405936","cluster_addr":"172.21.15.129:6811/3388405936","heartbeat_back_addr":"172.21.15.129:6815/3388405936","heartbeat_front_addr":"172.21.15.129:6813/3388405936","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.879825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.057602+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.575932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:24.170 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2024-09-14T06:51:24.170 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T06:51:24.480 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:24.481 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":15,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T06:51:23.104554+0000","last_up_change":"2024-09-14T06:51:14.927617+0000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:15.054238+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T06:51:19.403913+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6817","nonce":3974496147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6819","nonce":3974496147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6823","nonce":3974496147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6821","nonce":3974496147}]},"public_addr":"172.21.15.129:6817/3974496147","cluster_addr":"172.21.15.129:6819/3974496147","heartbeat_back_addr":"172.21.15.129:6823/3974496147","heartbeat_front_addr":"172.21.15.129:6821/3974496147","state":["exists","up"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6801","nonce":498324347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6803","nonce":498324347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6807","nonce":498324347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6805","nonce":498324347}]},"public_addr":"172.21.15.129:6801/498324347","cluster_addr":"172.21.15.129:6803/498324347","heartbeat_back_addr":"172.21.15.129:6807/498324347","heartbeat_front_addr":"172.21.15.129:6805/498324347","state":["exists","up"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6809","nonce":3388405936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6811","nonce":3388405936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6815","nonce":3388405936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6813","nonce":3388405936}]},"public_addr":"172.21.15.129:6809/3388405936","cluster_addr":"172.21.15.129:6811/3388405936","heartbeat_back_addr":"172.21.15.129:6815/3388405936","heartbeat_front_addr":"172.21.15.129:6813/3388405936","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.879825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.057602+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.575932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:24.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2024-09-14T06:51:24.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2024-09-14T06:51:24.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2024-09-14T06:51:24.658 INFO:teuthology.orchestra.run.smithi129.stdout:34359738372 2024-09-14T06:51:24.658 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-14T06:51:24.666 INFO:teuthology.orchestra.run.smithi129.stdout:34359738374 2024-09-14T06:51:24.667 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-14T06:51:24.668 INFO:teuthology.orchestra.run.smithi129.stdout:34359738373 2024-09-14T06:51:24.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-14T06:51:24.951 INFO:teuthology.orchestra.run.smithi129.stdout:34359738371 2024-09-14T06:51:24.968 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.0 2024-09-14T06:51:24.995 INFO:teuthology.orchestra.run.smithi129.stdout:34359738372 2024-09-14T06:51:25.002 INFO:teuthology.orchestra.run.smithi129.stdout:34359738373 2024-09-14T06:51:25.014 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738372 for osd.1 2024-09-14T06:51:25.020 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738373 for osd.2 2024-09-14T06:51:25.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-14T06:51:26.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-14T06:51:26.021 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-14T06:51:26.279 INFO:teuthology.orchestra.run.smithi129.stdout:34359738372 2024-09-14T06:51:26.297 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.0 2024-09-14T06:51:26.297 DEBUG:teuthology.parallel:result is None 2024-09-14T06:51:26.351 INFO:teuthology.orchestra.run.smithi129.stdout:34359738373 2024-09-14T06:51:26.368 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738373 for osd.1 2024-09-14T06:51:26.368 DEBUG:teuthology.parallel:result is None 2024-09-14T06:51:26.384 INFO:teuthology.orchestra.run.smithi129.stdout:34359738374 2024-09-14T06:51:26.400 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738374 for osd.2 2024-09-14T06:51:26.401 DEBUG:teuthology.parallel:result is None 2024-09-14T06:51:26.401 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-14T06:51:26.401 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T06:51:26.722 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:26.722 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T06:51:26.739 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-09-14T06:51:25.016534+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":14,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82644,"kb_used_data":1552,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032876,"statfs":{"total":289910292480,"available":289825665024,"internally_reserved":0,"allocated":1589248,"data_stored":1295959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.922478"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.124440+0000","last_change":"2024-09-14T06:51:23.124440+0000","last_active":"2024-09-14T06:51:23.124440+0000","last_peered":"2024-09-14T06:51:23.124440+0000","last_clean":"2024-09-14T06:51:23.124440+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T06:51:23.124440+0000","last_undegraded":"2024-09-14T06:51:23.124440+0000","last_fullsized":"2024-09-14T06:51:23.124440+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:54.063248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.124277+0000","last_change":"2024-09-14T06:51:23.124277+0000","last_active":"2024-09-14T06:51:23.124277+0000","last_peered":"2024-09-14T06:51:23.124277+0000","last_clean":"2024-09-14T06:51:23.124277+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T06:51:23.124277+0000","last_undegraded":"2024-09-14T06:51:23.124277+0000","last_fullsized":"2024-09-14T06:51:23.124277+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:36:43.389449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.123529+0000","last_change":"2024-09-14T06:51:23.123529+0000","last_active":"2024-09-14T06:51:23.123529+0000","last_peered":"2024-09-14T06:51:23.123529+0000","last_clean":"2024-09-14T06:51:23.123529+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T06:51:23.123529+0000","last_undegraded":"2024-09-14T06:51:23.123529+0000","last_fullsized":"2024-09-14T06:51:23.123529+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:18.274940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.123532+0000","last_change":"2024-09-14T06:51:23.123532+0000","last_active":"2024-09-14T06:51:23.123532+0000","last_peered":"2024-09-14T06:51:23.123532+0000","last_clean":"2024-09-14T06:51:23.123532+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T06:51:23.123532+0000","last_undegraded":"2024-09-14T06:51:23.123532+0000","last_fullsized":"2024-09-14T06:51:23.123532+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:21.958324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.120650+0000","last_change":"2024-09-14T06:51:23.118966+0000","last_active":"2024-09-14T06:51:23.120650+0000","last_peered":"2024-09-14T06:51:23.120650+0000","last_clean":"2024-09-14T06:51:23.120650+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T06:51:23.120650+0000","last_undegraded":"2024-09-14T06:51:23.120650+0000","last_fullsized":"2024-09-14T06:51:23.120650+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:59.628972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.113503+0000","last_change":"2024-09-14T06:51:23.113503+0000","last_active":"2024-09-14T06:51:23.113503+0000","last_peered":"2024-09-14T06:51:23.113503+0000","last_clean":"2024-09-14T06:51:23.113503+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T06:51:23.113503+0000","last_undegraded":"2024-09-14T06:51:23.113503+0000","last_fullsized":"2024-09-14T06:51:23.113503+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:15.559943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.113565+0000","last_change":"2024-09-14T06:51:23.113565+0000","last_active":"2024-09-14T06:51:23.113565+0000","last_peered":"2024-09-14T06:51:23.113565+0000","last_clean":"2024-09-14T06:51:23.113565+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T06:51:23.113565+0000","last_undegraded":"2024-09-14T06:51:23.113565+0000","last_fullsized":"2024-09-14T06:51:23.113565+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:39.862806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.123208+0000","last_change":"2024-09-14T06:51:23.123208+0000","last_active":"2024-09-14T06:51:23.123208+0000","last_peered":"2024-09-14T06:51:23.123208+0000","last_clean":"2024-09-14T06:51:23.123208+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T06:51:23.123208+0000","last_undegraded":"2024-09-14T06:51:23.123208+0000","last_fullsized":"2024-09-14T06:51:23.123208+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:58.546304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":88,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.121816+0000","last_change":"2024-09-14T06:51:17.941930+0000","last_active":"2024-09-14T06:51:23.121816+0000","last_peered":"2024-09-14T06:51:23.121816+0000","last_clean":"2024-09-14T06:51:23.121816+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T06:51:23.121816+0000","last_undegraded":"2024-09-14T06:51:23.121816+0000","last_fullsized":"2024-09-14T06:51:23.121816+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_clean_scrub_stamp":"2024-09-14T06:51:16.017137+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:40.900521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738374,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":131072,"data_stored":38395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T06:51:26.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T06:51:27.032 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:27.032 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T06:51:27.048 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2024-09-14T06:51:27.016901+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82644,"kb_used_data":1552,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032876,"statfs":{"total":289910292480,"available":289825665024,"internally_reserved":0,"allocated":1589248,"data_stored":1295959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001336"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.124440+0000","last_change":"2024-09-14T06:51:23.124440+0000","last_active":"2024-09-14T06:51:23.124440+0000","last_peered":"2024-09-14T06:51:23.124440+0000","last_clean":"2024-09-14T06:51:23.124440+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T06:51:23.124440+0000","last_undegraded":"2024-09-14T06:51:23.124440+0000","last_fullsized":"2024-09-14T06:51:23.124440+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:54.063248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.124277+0000","last_change":"2024-09-14T06:51:23.124277+0000","last_active":"2024-09-14T06:51:23.124277+0000","last_peered":"2024-09-14T06:51:23.124277+0000","last_clean":"2024-09-14T06:51:23.124277+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T06:51:23.124277+0000","last_undegraded":"2024-09-14T06:51:23.124277+0000","last_fullsized":"2024-09-14T06:51:23.124277+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:36:43.389449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.123529+0000","last_change":"2024-09-14T06:51:23.123529+0000","last_active":"2024-09-14T06:51:23.123529+0000","last_peered":"2024-09-14T06:51:23.123529+0000","last_clean":"2024-09-14T06:51:23.123529+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T06:51:23.123529+0000","last_undegraded":"2024-09-14T06:51:23.123529+0000","last_fullsized":"2024-09-14T06:51:23.123529+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:18.274940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.123532+0000","last_change":"2024-09-14T06:51:23.123532+0000","last_active":"2024-09-14T06:51:23.123532+0000","last_peered":"2024-09-14T06:51:23.123532+0000","last_clean":"2024-09-14T06:51:23.123532+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T06:51:23.123532+0000","last_undegraded":"2024-09-14T06:51:23.123532+0000","last_fullsized":"2024-09-14T06:51:23.123532+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:21.958324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.120650+0000","last_change":"2024-09-14T06:51:23.118966+0000","last_active":"2024-09-14T06:51:23.120650+0000","last_peered":"2024-09-14T06:51:23.120650+0000","last_clean":"2024-09-14T06:51:23.120650+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T06:51:23.120650+0000","last_undegraded":"2024-09-14T06:51:23.120650+0000","last_fullsized":"2024-09-14T06:51:23.120650+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:59.628972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.113503+0000","last_change":"2024-09-14T06:51:23.113503+0000","last_active":"2024-09-14T06:51:23.113503+0000","last_peered":"2024-09-14T06:51:23.113503+0000","last_clean":"2024-09-14T06:51:23.113503+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T06:51:23.113503+0000","last_undegraded":"2024-09-14T06:51:23.113503+0000","last_fullsized":"2024-09-14T06:51:23.113503+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:15.559943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.113565+0000","last_change":"2024-09-14T06:51:23.113565+0000","last_active":"2024-09-14T06:51:23.113565+0000","last_peered":"2024-09-14T06:51:23.113565+0000","last_clean":"2024-09-14T06:51:23.113565+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T06:51:23.113565+0000","last_undegraded":"2024-09-14T06:51:23.113565+0000","last_fullsized":"2024-09-14T06:51:23.113565+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:39.862806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.123208+0000","last_change":"2024-09-14T06:51:23.123208+0000","last_active":"2024-09-14T06:51:23.123208+0000","last_peered":"2024-09-14T06:51:23.123208+0000","last_clean":"2024-09-14T06:51:23.123208+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T06:51:23.123208+0000","last_undegraded":"2024-09-14T06:51:23.123208+0000","last_fullsized":"2024-09-14T06:51:23.123208+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:58.546304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":88,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-14T06:51:23.121816+0000","last_change":"2024-09-14T06:51:17.941930+0000","last_active":"2024-09-14T06:51:23.121816+0000","last_peered":"2024-09-14T06:51:23.121816+0000","last_clean":"2024-09-14T06:51:23.121816+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T06:51:23.121816+0000","last_undegraded":"2024-09-14T06:51:23.121816+0000","last_fullsized":"2024-09-14T06:51:23.121816+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_clean_scrub_stamp":"2024-09-14T06:51:16.017137+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:40.900521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":131072,"data_stored":38395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T06:51:27.049 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-14T06:51:27.049 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2024-09-14T06:51:27.049 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2024-09-14T06:51:27.049 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2024-09-14T06:51:27.363 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:27.363 INFO:teuthology.orchestra.run.smithi129.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T06:51:27.381 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2024-09-14T06:51:27.381 INFO:teuthology.run_tasks:Running task ceph... 2024-09-14T06:51:27.391 INFO:tasks.ceph:Creating ceph cluster cluster2... 2024-09-14T06:51:27.391 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-14T06:51:27.391 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398', 'branch': 'main', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7900398', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 950, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 5839, 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47', 'sleep_before_teardown': 0, 'subset': '99/128', 'suite': 'rbd', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_0d61375b8ab53be4c233f5b93e463edfdff9316a/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '0d61375b8ab53be4c233f5b93e463edfdff9316a', 'targets': {'smithi129.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCewASal6XC5348xuSGBkugXPsEDpkJbeCgS3/Cs+587P8hWBK2W/EOtSErdQ4R+RR1SNAalWl4DZy/nL/NQoSw=', 'smithi169.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSdrVcJWaQlujfB2Sjg4WwVO1/3Ud6hUEmt60A7ZtRUWilyOrIDw3caPOYXMYTtwBVS3Qu54tSO7pau77hUBbo='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '7f24e589328ae9e639c0009d93340e5b235d0a47'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-10_20:16:17', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-14T06:51:27.392 DEBUG:teuthology.orchestra.run.smithi129:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-14T06:51:27.394 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-14T06:51:27.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T06:51:27.441 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m0777 -- /var/run/ceph 2024-09-14T06:51:27.456 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:27.456 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T06:51:27.503 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T06:51:27.503 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_1 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 882 Links: 1 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-09-14 06:45:18.620017364 +0000 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-09-14 06:45:18.616017400 +0000 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-09-14 06:45:18.616017400 +0000 2024-09-14T06:51:27.551 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2024-09-14T06:51:27.552 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T06:51:27.605 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-09-14T06:51:27.605 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-09-14T06:51:27.606 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000379693 s, 1.3 MB/s 2024-09-14T06:51:27.607 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T06:51:27.657 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_2 2024-09-14T06:51:27.703 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T06:51:27.703 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:51:27.703 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 896 Links: 1 2024-09-14T06:51:27.703 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:51:27.703 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-09-14 06:45:19.320011112 +0000 2024-09-14T06:51:27.703 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-09-14 06:45:19.068013363 +0000 2024-09-14T06:51:27.704 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-09-14 06:45:19.068013363 +0000 2024-09-14T06:51:27.704 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2024-09-14T06:51:27.704 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T06:51:27.757 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-09-14T06:51:27.757 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-09-14T06:51:27.757 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000423964 s, 1.2 MB/s 2024-09-14T06:51:27.759 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T06:51:27.809 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_3 2024-09-14T06:51:27.855 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T06:51:27.855 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:51:27.855 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 918 Links: 1 2024-09-14T06:51:27.855 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:51:27.856 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-09-14 06:45:19.804006789 +0000 2024-09-14T06:51:27.856 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-09-14 06:45:19.560008969 +0000 2024-09-14T06:51:27.856 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-09-14 06:45:19.560008969 +0000 2024-09-14T06:51:27.856 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2024-09-14T06:51:27.856 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T06:51:27.910 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-09-14T06:51:27.910 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-09-14T06:51:27.910 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000446483 s, 1.1 MB/s 2024-09-14T06:51:27.911 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T06:51:27.960 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_4 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 5h/5d Inode: 932 Links: 1 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2024-09-14 06:45:20.312002251 +0000 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2024-09-14 06:45:20.036004717 +0000 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2024-09-14 06:45:20.036004717 +0000 2024-09-14T06:51:28.007 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2024-09-14T06:51:28.008 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T06:51:28.061 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2024-09-14T06:51:28.061 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2024-09-14T06:51:28.062 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000385984 s, 1.3 MB/s 2024-09-14T06:51:28.063 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T06:51:28.112 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:51:28.112 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi169.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T06:51:28.112 INFO:tasks.ceph:Generating config... 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client] log to stderr = False 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.114 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.115 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.115 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.115 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-14T06:51:28.115 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-14T06:51:28.115 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-14T06:51:28.115 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-14T06:51:28.115 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-14T06:51:28.115 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-14T06:51:28.115 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-14T06:51:28.115 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-14T06:51:28.115 INFO:tasks.ceph:[osd] bdev async discard = True 2024-09-14T06:51:28.115 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-09-14T06:51:28.115 INFO:tasks.ceph:[osd] bluestore allocator = stupid 2024-09-14T06:51:28.115 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-14T06:51:28.116 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-14T06:51:28.116 INFO:tasks.ceph:Setting up cluster2.mon.a... 2024-09-14T06:51:28.117 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.185 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.188 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.266 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.322 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '172.21.15.169')] 2024-09-14T06:51:28.322 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.169', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-14T06:51:28.323 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:28.323 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T06:51:28.367 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.169 --print /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:monmaptool: generated fsid 798b1ebb-0651-4c92-aba2-17acca9548ca 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:setting min_mon_release = quincy 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:epoch 0 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:fsid 798b1ebb-0651-4c92-aba2-17acca9548ca 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:last_changed 2024-09-14T06:51:28.449180+0000 2024-09-14T06:51:28.447 INFO:teuthology.orchestra.run.smithi169.stdout:created 2024-09-14T06:51:28.449180+0000 2024-09-14T06:51:28.448 INFO:teuthology.orchestra.run.smithi169.stdout:min_mon_release 17 (quincy) 2024-09-14T06:51:28.448 INFO:teuthology.orchestra.run.smithi169.stdout:election_strategy: 1 2024-09-14T06:51:28.448 INFO:teuthology.orchestra.run.smithi169.stdout:0: [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] mon.a 2024-09-14T06:51:28.448 INFO:teuthology.orchestra.run.smithi169.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2024-09-14T06:51:28.449 DEBUG:teuthology.orchestra.run.smithi169:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-14T06:51:28.495 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 798b1ebb-0651-4c92-aba2-17acca9548ca... 2024-09-14T06:51:28.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-14T06:51:28.498 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout:[global] 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: chdir = "" 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: auth supported = cephx 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: filestore xattr use omap = true 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: mon clock drift allowed = 1.000 2024-09-14T06:51:28.541 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: osd crush chooseleaf type = 0 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: auth debug = true 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: ms die on old message = true 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: ms die on bug = true 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: mon pg warn max object skew = 0 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: osd pool default size = 2 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.542 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd allow primary affinity = true 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd allow pg remap = true 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on legacy crush tunables = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on crush straw calc version zero = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on no sortbitwise = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on osd down out interval zero = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on too few osds = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: mon_allow_pool_size_one = true 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.543 INFO:teuthology.orchestra.run.smithi129.stdout: osd default data pool replay window = 5 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: mon allow pool delete = true 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: mon cluster log file level = debug 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: debug asserts on shutdown = true 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: mon health detail to clog = false 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: mon host = 172.21.15.169 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: mon client directed command retry = 5 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: mon warn on pool no app = False 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: ms inject socket failures = 5000 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: fsid = 798b1ebb-0651-4c92-aba2-17acca9548ca 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.544 INFO:teuthology.orchestra.run.smithi129.stdout:[osd] 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd journal size = 100 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd scrub load threshold = 5.0 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd scrub max interval = 600 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd mclock profile = high_recovery_ops 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd recover clone overlap = true 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd recovery max chunk = 1048576 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.545 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug shutdown = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug op order = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug verify stray on activate = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug trim objects = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd open classes on start = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug pg log writeout = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd deep scrub update digest min age = 30 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: osd map max advance = 10 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: journal zero on create = true 2024-09-14T06:51:28.546 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: filestore ondisk finisher threads = 3 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: filestore apply finisher threads = 3 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: bdev debug aio = true 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: osd debug misdirected ops = true 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: bdev async discard = True 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: bdev enable discard = True 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: bluestore allocator = stupid 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: bluestore block size = 96636764160 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: bluestore fsck on mount = True 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: debug bluefs = 1/20 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: debug bluestore = 1/20 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: debug ms = 1 2024-09-14T06:51:28.547 INFO:teuthology.orchestra.run.smithi129.stdout: debug osd = 20 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: debug rocksdb = 4/10 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd full ratio = 0.9 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd nearfull ratio = 0.8 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: osd failsafe full ratio = 0.95 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: osd objectstore = bluestore 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout:[mgr] 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: debug ms = 1 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: debug mgr = 20 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: debug mon = 20 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: debug auth = 20 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:51:28.548 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mgr/telemetry/nag = false 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout:[mon] 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: debug ms = 1 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: debug mon = 20 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: debug paxos = 20 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: debug auth = 20 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mon data avail warn = 5 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mon mgr mkfs grace = 240 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd reporter subtree level = osd 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mon osd prime pg temp = true 2024-09-14T06:51:28.549 INFO:teuthology.orchestra.run.smithi129.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: # 1m isn't quite enough 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: mon_down_mkfs_grace = 2m 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: mon_warn_on_filestore_osds = false 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T06:51:28.550 INFO:teuthology.orchestra.run.smithi129.stdout:[client] 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: rgw cache enabled = true 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: rgw enable ops log = true 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: rgw enable usage log = true 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: debug rbd = 20 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: debug rbd_mirror = 15 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: log to stderr = False 2024-09-14T06:51:28.551 INFO:teuthology.orchestra.run.smithi129.stdout: rbd default features = 125 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: rbd mirror image policy type = simple 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout:[mon.a] 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.0] 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.1] 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.2] 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.552 INFO:teuthology.orchestra.run.smithi129.stdout:[client.mirror.3] 2024-09-14T06:51:28.553 INFO:teuthology.orchestra.run.smithi129.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.553 INFO:teuthology.orchestra.run.smithi129.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.569 INFO:teuthology.orchestra.run.smithi169.stdout:[global] 2024-09-14T06:51:28.569 INFO:teuthology.orchestra.run.smithi169.stdout: chdir = "" 2024-09-14T06:51:28.569 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-14T06:51:28.569 INFO:teuthology.orchestra.run.smithi169.stdout: auth supported = cephx 2024-09-14T06:51:28.569 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: filestore xattr use omap = true 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: mon clock drift allowed = 1.000 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: osd crush chooseleaf type = 0 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: auth debug = true 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: ms die on old message = true 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: ms die on bug = true 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: mon pg warn max object skew = 0 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.570 INFO:teuthology.orchestra.run.smithi169.stdout: # disable pg_autoscaler by default for new pools 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: osd pool default size = 2 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd allow primary affinity = true 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd allow pg remap = true 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on legacy crush tunables = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on crush straw calc version zero = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on no sortbitwise = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on osd down out interval zero = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on too few osds = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: mon_allow_pool_size_one = true 2024-09-14T06:51:28.571 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: osd default data pool replay window = 5 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: mon allow pool delete = true 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: mon cluster log file level = debug 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: debug asserts on shutdown = true 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: mon health detail to clog = false 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: mon host = 172.21.15.169 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: mon client directed command retry = 5 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on pool no app = False 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: ms inject socket failures = 5000 2024-09-14T06:51:28.572 INFO:teuthology.orchestra.run.smithi169.stdout: fsid = 798b1ebb-0651-4c92-aba2-17acca9548ca 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout:[osd] 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd journal size = 100 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd scrub load threshold = 5.0 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd scrub max interval = 600 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd mclock profile = high_recovery_ops 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd recover clone overlap = true 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd recovery max chunk = 1048576 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug shutdown = true 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug op order = true 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug verify stray on activate = true 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug trim objects = true 2024-09-14T06:51:28.573 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: osd open classes on start = true 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug pg log writeout = true 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: osd deep scrub update digest min age = 30 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: osd map max advance = 10 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: journal zero on create = true 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: filestore ondisk finisher threads = 3 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: filestore apply finisher threads = 3 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: bdev debug aio = true 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug misdirected ops = true 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: bdev async discard = True 2024-09-14T06:51:28.574 INFO:teuthology.orchestra.run.smithi169.stdout: bdev enable discard = True 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: bluestore allocator = stupid 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: bluestore block size = 96636764160 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: bluestore fsck on mount = True 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: debug bluefs = 1/20 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: debug bluestore = 1/20 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: debug osd = 20 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: debug rocksdb = 4/10 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd backfillfull_ratio = 0.85 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd full ratio = 0.9 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd nearfull ratio = 0.8 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: osd failsafe full ratio = 0.95 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: osd objectstore = bluestore 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout:[mgr] 2024-09-14T06:51:28.575 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug mgr = 20 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug mon = 20 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug auth = 20 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: mgr/telemetry/nag = false 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout:[mon] 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug mon = 20 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug paxos = 20 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: debug auth = 20 2024-09-14T06:51:28.576 INFO:teuthology.orchestra.run.smithi169.stdout: mon data avail warn = 5 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon mgr mkfs grace = 240 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min pgs per osd = 4 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd reporter subtree level = osd 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd prime pg temp = true 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min bytes per osd = 10 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: auth mon ticket ttl = 660 # 11m 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: auth service ticket ttl = 240 # 4m 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: # don't complain about insecure global_id in the test suite 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.577 INFO:teuthology.orchestra.run.smithi169.stdout: # 1m isn't quite enough 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: mon_down_mkfs_grace = 2m 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_filestore_osds = false 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout:[client] 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: rgw cache enabled = true 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: rgw enable ops log = true 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: rgw enable usage log = true 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: debug rbd = 20 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: debug rbd_mirror = 15 2024-09-14T06:51:28.578 INFO:teuthology.orchestra.run.smithi169.stdout: log to stderr = False 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: rbd default features = 125 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: rbd mirror image policy type = simple 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout:[mon.a] 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.0] 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.1] 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.2] 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.3] 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-14T06:51:28.579 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-14T06:51:28.580 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2024-09-14T06:51:28.580 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.671 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-14T06:51:28.672 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:28.672 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2024-09-14T06:51:28.677 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:28.677 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2024-09-14T06:51:28.723 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T06:51:28.723 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:28.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-14T06:51:28.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.746 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:28.747 DEBUG:teuthology.orchestra.run.smithi129:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:28.792 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:51:28.792 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:28.792 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-14T06:51:28.793 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-14T06:51:28.815 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:28.815 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:28.863 INFO:tasks.ceph:Setting up mon nodes... 2024-09-14T06:51:28.863 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-14T06:51:28.864 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-14T06:51:28.908 INFO:teuthology.orchestra.run.smithi129.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-14T06:51:28.911 INFO:tasks.ceph:Setting up mds nodes... 2024-09-14T06:51:28.911 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-14T06:51:28.911 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2024-09-14T06:51:28.944 INFO:teuthology.orchestra.run.smithi129.stdout:creating /etc/ceph/cluster2.client.0.keyring 2024-09-14T06:51:28.954 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2024-09-14T06:51:28.991 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2024-09-14T06:51:29.002 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-14T06:51:29.077 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-14T06:51:29.088 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-14T06:51:29.161 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-14T06:51:29.172 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-14T06:51:29.248 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-14T06:51:29.259 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-14T06:51:29.333 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-14T06:51:29.344 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-14T06:51:29.416 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-14T06:51:29.425 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-14T06:51:29.498 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-14T06:51:29.509 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-14T06:51:29.581 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-14T06:51:29.590 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-14T06:51:29.591 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi129.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi169.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-14T06:51:29.591 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2024-09-14T06:51:29.644 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:51:29.644 INFO:tasks.ceph:role: cluster2.osd.0 2024-09-14T06:51:29.644 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:51:29.644 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-14T06:51:29.707 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T06:51:29.707 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T06:51:29.707 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T06:51:29.707 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-14T06:51:29.708 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T06:51:29.708 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2024-09-14T06:51:29.708 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T06:51:29.708 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-14T06:51:29.708 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T06:51:29.708 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T06:51:30.182 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2024-09-14T06:51:30.185 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2024-09-14T06:51:30.185 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2024-09-14T06:51:30.319 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2024-09-14T06:51:30.329 INFO:teuthology.orchestra.run.smithi169.stderr:sudo: /sbin/restorecon: command not found 2024-09-14T06:51:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:51:30.330 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2024-09-14T06:51:30.387 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:51:30.387 INFO:tasks.ceph:role: cluster2.osd.1 2024-09-14T06:51:30.388 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:51:30.388 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T06:51:30.451 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-14T06:51:30.452 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T06:51:30.452 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T06:51:30.878 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2024-09-14T06:51:30.880 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2024-09-14T06:51:30.881 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2024-09-14T06:51:30.943 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2024-09-14T06:51:30.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:51:30.997 INFO:teuthology.orchestra.run.smithi169.stderr:sudo: /sbin/restorecon: command not found 2024-09-14T06:51:30.997 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2024-09-14T06:51:31.051 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-14T06:51:31.051 INFO:tasks.ceph:role: cluster2.osd.2 2024-09-14T06:51:31.051 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:51:31.051 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-14T06:51:31.115 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-14T06:51:31.116 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-14T06:51:31.590 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2024-09-14T06:51:31.592 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2024-09-14T06:51:31.593 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2024-09-14T06:51:31.689 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2024-09-14T06:51:31.700 INFO:teuthology.orchestra.run.smithi169.stderr:sudo: /sbin/restorecon: command not found 2024-09-14T06:51:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:51:31.701 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:31.814 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:31.812+0000 7f17982277c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2024-09-14T06:51:31.818 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:31.816+0000 7f17982277c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2024-09-14T06:51:31.818 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:31.816+0000 7f17982277c0 -1 bdev(0x55ecd2dbf800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2024-09-14T06:51:31.818 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:31.816+0000 7f17982277c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2024-09-14T06:51:34.963 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2024-09-14T06:51:34.977 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:35.058 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:35.056+0000 7fe10522b7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2024-09-14T06:51:35.059 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:35.060+0000 7fe10522b7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2024-09-14T06:51:35.059 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:35.060+0000 7fe10522b7c0 -1 bdev(0x564de3bcf800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2024-09-14T06:51:35.059 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:35.060+0000 7fe10522b7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2024-09-14T06:51:38.193 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2024-09-14T06:51:38.247 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:38.330 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:38.328+0000 7fb2538007c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2024-09-14T06:51:38.331 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:38.332+0000 7fb2538007c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2024-09-14T06:51:38.331 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:38.332+0000 7fb2538007c0 -1 bdev(0x55e021ddd800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2024-09-14T06:51:38.331 INFO:teuthology.orchestra.run.smithi169.stderr:2024-09-14T06:51:38.332+0000 7fb2538007c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2024-09-14T06:51:41.478 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2024-09-14T06:51:41.532 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-14T06:51:41.532 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:41.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2024-09-14T06:51:41.546 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.547 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2024-09-14T06:51:41.585 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.585 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2024-09-14T06:51:41.637 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.638 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2024-09-14T06:51:41.690 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T06:51:41.690 DEBUG:teuthology.orchestra.run.smithi129:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2024-09-14T06:51:41.696 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.696 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2024-09-14T06:51:41.735 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.735 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2024-09-14T06:51:41.784 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.784 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2024-09-14T06:51:41.831 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.831 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2024-09-14T06:51:41.879 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.880 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2024-09-14T06:51:41.927 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.927 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2024-09-14T06:51:41.975 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:41.975 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2024-09-14T06:51:42.023 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:42.023 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2024-09-14T06:51:42.071 INFO:tasks.ceph:Adding keys to all mons... 2024-09-14T06:51:42.071 DEBUG:teuthology.orchestra.run.smithi169:> sudo tee -a /etc/ceph/cluster2.keyring 2024-09-14T06:51:42.125 INFO:teuthology.orchestra.run.smithi169.stdout:[mgr.x] 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBwMuVmtGhRNhAAcwCU27XLVvCa29OScCkO0g== 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.0] 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBzMuVmwgypMBAADzwgqEvSqmvg27p8EWh0xA== 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.1] 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQB3MuVmWsadAxAAn4ABXLojnXmxO92k+zfGIQ== 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.2] 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQB6MuVmF5vQExAAKOYjtdwBlJZHEMXOujfc8A== 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout:[client.0] 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBwMuVm7S5zOBAAqYPBwT6kR9KDOuflgtCZjg== 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror] 2024-09-14T06:51:42.126 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBwMuVmfRo2OxAA94ze85NKZfFzClcFY+HDLA== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.0] 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmTQDBBBAArr0BZHc25YiUHJfhNh8ySQ== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.1] 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVma5+/CRAAnMJyQbBZs6/0HcklUKwakg== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.2] 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmoantDhAAdkXZBMCvImQGjTdIpI2ZIA== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.3] 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmDXz4ExAA7jLLw9omp2Y4EEz/ttKAvw== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.4] 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmoz3tGBAAWNSJs8uVaUceCR+iORxUXA== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.5] 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmgzPbHRAAFbawzYO0GFxHzwTXmm2A+Q== 2024-09-14T06:51:42.127 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.6] 2024-09-14T06:51:42.128 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmdWe+IhAAwXlIHlh4e6j8Oj0DNAZmDQ== 2024-09-14T06:51:42.128 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-14T06:51:42.219 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T06:51:42.307 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T06:51:42.346 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-14T06:51:42.430 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.516 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.558 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.650 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.697 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.739 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.825 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.909 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:42.998 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-14T06:51:43.041 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-14T06:51:43.041 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2024-09-14T06:51:43.094 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2024-09-14T06:51:43.184 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2024-09-14T06:51:43.194 DEBUG:teuthology.orchestra.run.smithi169:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2024-09-14T06:51:43.238 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2024-09-14T06:51:43.239 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2024-09-14T06:51:43.239 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2024-09-14T06:51:43.283 INFO:tasks.ceph.cluster2.mon.a:Started 2024-09-14T06:51:43.284 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2024-09-14T06:51:43.284 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2024-09-14T06:51:43.284 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2024-09-14T06:51:43.287 INFO:tasks.ceph.cluster2.mgr.x:Started 2024-09-14T06:51:43.287 DEBUG:tasks.ceph:set 0 configs 2024-09-14T06:51:43.287 INFO:tasks.ceph:Setting crush tunables to default 2024-09-14T06:51:43.287 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster cluster2 osd crush tunables default 2024-09-14T06:51:43.519 INFO:teuthology.orchestra.run.smithi169.stderr:adjusted tunables profile to default 2024-09-14T06:51:43.535 INFO:tasks.ceph:check_enable_crimson: False 2024-09-14T06:51:43.535 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2024-09-14T06:51:43.536 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:43.536 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2024-09-14T06:51:43.546 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:43.546 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2024-09-14T06:51:43.596 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:51:43.596 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2024-09-14T06:51:43.649 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster cluster2 osd new 82d699dd-3b35-4ab1-87a9-d69ce7d632eb 0 2024-09-14T06:51:43.895 INFO:teuthology.orchestra.run.smithi169.stdout:0 2024-09-14T06:51:43.913 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster cluster2 osd new 0cf7e906-2a01-4c03-89a9-e4056eb15813 1 2024-09-14T06:51:44.119 INFO:teuthology.orchestra.run.smithi169.stdout:1 2024-09-14T06:51:44.137 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster cluster2 osd new 9dcade49-882f-40b8-b3a0-f14d759427c4 2 2024-09-14T06:51:44.339 INFO:teuthology.orchestra.run.smithi169.stdout:2 2024-09-14T06:51:44.357 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2024-09-14T06:51:44.357 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2024-09-14T06:51:44.359 INFO:tasks.ceph.cluster2.osd.0:Started 2024-09-14T06:51:44.359 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2024-09-14T06:51:44.360 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2024-09-14T06:51:44.361 INFO:tasks.ceph.cluster2.osd.1:Started 2024-09-14T06:51:44.361 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2024-09-14T06:51:44.362 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2024-09-14T06:51:44.363 INFO:tasks.ceph.cluster2.osd.2:Started 2024-09-14T06:51:44.364 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T06:51:44.552 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:51:44.552 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":5,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T06:51:44.340418+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:44.569 INFO:tasks.ceph.ceph_manager.cluster2:[] 2024-09-14T06:51:44.569 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-14T06:51:44.873 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-14T06:51:45.066 INFO:teuthology.misc.health.smithi169.stdout: 2024-09-14T06:51:45.067 INFO:teuthology.misc.health.smithi169.stdout:{"epoch":5,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T06:51:44.340418+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:45.081 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T06:51:45.363 INFO:tasks.ceph.cluster2.mgr.x.smithi129.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2024-09-14T06:51:45.363 INFO:tasks.ceph.cluster2.mgr.x.smithi129.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2024-09-14T06:51:45.363 INFO:tasks.ceph.cluster2.mgr.x.smithi129.stderr: from numpy import show_config as show_numpy_config 2024-09-14T06:51:45.492 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T06:51:45.492+0000 7f7c460157c0 -1 Falling back to public interface 2024-09-14T06:51:45.499 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T06:51:45.500+0000 7f7dfbeca7c0 -1 Falling back to public interface 2024-09-14T06:51:45.732 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:51:45.732+0000 7fb4093b27c0 -1 Falling back to public interface 2024-09-14T06:51:47.915 INFO:tasks.ceph.cluster2.mgr.x.smithi129.stderr:2024-09-14T06:51:47.912+0000 7fed9a5f8640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T06:51:49.442 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T06:51:49.444+0000 7f7c460157c0 -1 osd.2 0 log_to_monitors true 2024-09-14T06:51:49.623 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T06:51:49.624+0000 7f7dfbeca7c0 -1 osd.1 0 log_to_monitors true 2024-09-14T06:51:50.126 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:51:50.124+0000 7fb4093b27c0 -1 osd.0 0 log_to_monitors true 2024-09-14T06:51:51.385 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-14T06:51:51.727 INFO:teuthology.misc.health.smithi169.stdout: 2024-09-14T06:51:51.727 INFO:teuthology.misc.health.smithi169.stdout:{"epoch":7,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T06:51:50.971391+0000","last_up_change":"0.000000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:51.745 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-14T06:51:52.786 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T06:51:52.784+0000 7f7c41fbe640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T06:51:52.815 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T06:51:52.816+0000 7f7c39df0640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T06:51:52.828 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T06:51:52.828+0000 7f7df7e73640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T06:51:52.867 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T06:51:52.868+0000 7f7defca5640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T06:51:53.235 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:51:53.236+0000 7fb40535b640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T06:51:53.267 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:51:53.268+0000 7fb3fd18d640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T06:51:58.048 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-14T06:51:58.368 INFO:teuthology.misc.health.smithi169.stdout: 2024-09-14T06:51:58.368 INFO:teuthology.misc.health.smithi169.stdout:{"epoch":12,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T06:51:57.016850+0000","last_up_change":"2024-09-14T06:51:53.831394+0000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:54.956943+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6817","nonce":2226319647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6819","nonce":2226319647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6823","nonce":2226319647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6821","nonce":2226319647}]},"public_addr":"172.21.15.169:6817/2226319647","cluster_addr":"172.21.15.169:6819/2226319647","heartbeat_back_addr":"172.21.15.169:6823/2226319647","heartbeat_front_addr":"172.21.15.169:6821/2226319647","state":["exists","up"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6809","nonce":324375027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6811","nonce":324375027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6815","nonce":324375027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6813","nonce":324375027}]},"public_addr":"172.21.15.169:6809/324375027","cluster_addr":"172.21.15.169:6811/324375027","heartbeat_back_addr":"172.21.15.169:6815/324375027","heartbeat_front_addr":"172.21.15.169:6813/324375027","state":["exists","up"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6801","nonce":126512792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6803","nonce":126512792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6807","nonce":126512792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6805","nonce":126512792}]},"public_addr":"172.21.15.169:6801/126512792","cluster_addr":"172.21.15.169:6803/126512792","heartbeat_back_addr":"172.21.15.169:6807/126512792","heartbeat_front_addr":"172.21.15.169:6805/126512792","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:51.167791+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.668476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.467076+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:51:58.386 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-14T06:51:58.386 INFO:tasks.ceph:Creating RBD pool 2024-09-14T06:51:58.386 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster cluster2 osd pool create rbd 8 2024-09-14T06:51:59.033 INFO:teuthology.orchestra.run.smithi169.stderr:pool 'rbd' created 2024-09-14T06:51:59.058 DEBUG:teuthology.orchestra.run.smithi169:> rbd --cluster cluster2 pool init rbd 2024-09-14T06:52:02.072 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2024-09-14T06:52:02.072 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2024-09-14T06:52:02.073 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-14T06:52:02.391 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:02.410 INFO:teuthology.orchestra.run.smithi169.stdout:[{"version":4,"timestamp":"2024-09-14T06:51:53.237843+0000","name":"","changes":[{"name":"osd.0/osd_mclock_max_capacity_iops_ssd","new_value":"4557.416092"}]}] 2024-09-14T06:52:02.411 INFO:tasks.ceph_manager:config epoch is 4 2024-09-14T06:52:02.411 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2024-09-14T06:52:02.411 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2024-09-14T06:52:02.411 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2024-09-14T06:52:02.765 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:02.783 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":5,"flags":0,"active_gid":4100,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":1218189969},{"type":"v1","addr":"172.21.15.129:6827","nonce":1218189969}]},"active_addr":"172.21.15.129:6827/1218189969","active_change":"2024-09-14T06:51:46.903325+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":2244640089}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":934458612}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":2689580082}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.129:0","nonce":1955085153}]}]} 2024-09-14T06:52:02.785 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2024-09-14T06:52:02.785 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2024-09-14T06:52:02.786 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T06:52:03.089 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:03.089 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":16,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T06:52:02.046938+0000","last_up_change":"2024-09-14T06:51:53.831394+0000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:54.956943+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T06:51:58.692382+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6817","nonce":2226319647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6819","nonce":2226319647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6823","nonce":2226319647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6821","nonce":2226319647}]},"public_addr":"172.21.15.169:6817/2226319647","cluster_addr":"172.21.15.169:6819/2226319647","heartbeat_back_addr":"172.21.15.169:6823/2226319647","heartbeat_front_addr":"172.21.15.169:6821/2226319647","state":["exists","up"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6809","nonce":324375027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6811","nonce":324375027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6815","nonce":324375027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6813","nonce":324375027}]},"public_addr":"172.21.15.169:6809/324375027","cluster_addr":"172.21.15.169:6811/324375027","heartbeat_back_addr":"172.21.15.169:6815/324375027","heartbeat_front_addr":"172.21.15.169:6813/324375027","state":["exists","up"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6801","nonce":126512792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6803","nonce":126512792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6807","nonce":126512792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6805","nonce":126512792}]},"public_addr":"172.21.15.169:6801/126512792","cluster_addr":"172.21.15.169:6803/126512792","heartbeat_back_addr":"172.21.15.169:6807/126512792","heartbeat_front_addr":"172.21.15.169:6805/126512792","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:51.167791+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.668476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.467076+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:52:03.106 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2024-09-14T06:52:03.106 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T06:52:03.406 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:03.406 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":16,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T06:52:02.046938+0000","last_up_change":"2024-09-14T06:51:53.831394+0000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:54.956943+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T06:51:58.692382+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6817","nonce":2226319647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6819","nonce":2226319647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6823","nonce":2226319647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6821","nonce":2226319647}]},"public_addr":"172.21.15.169:6817/2226319647","cluster_addr":"172.21.15.169:6819/2226319647","heartbeat_back_addr":"172.21.15.169:6823/2226319647","heartbeat_front_addr":"172.21.15.169:6821/2226319647","state":["exists","up"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6809","nonce":324375027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6811","nonce":324375027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6815","nonce":324375027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6813","nonce":324375027}]},"public_addr":"172.21.15.169:6809/324375027","cluster_addr":"172.21.15.169:6811/324375027","heartbeat_back_addr":"172.21.15.169:6815/324375027","heartbeat_front_addr":"172.21.15.169:6813/324375027","state":["exists","up"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6801","nonce":126512792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6803","nonce":126512792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6807","nonce":126512792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6805","nonce":126512792}]},"public_addr":"172.21.15.169:6801/126512792","cluster_addr":"172.21.15.169:6803/126512792","heartbeat_back_addr":"172.21.15.169:6807/126512792","heartbeat_front_addr":"172.21.15.169:6805/126512792","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:51.167791+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.668476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.467076+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T06:52:03.424 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2024-09-14T06:52:03.425 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2024-09-14T06:52:03.425 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2024-09-14T06:52:03.591 INFO:teuthology.orchestra.run.smithi169.stdout:38654705669 2024-09-14T06:52:03.592 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-14T06:52:03.593 INFO:teuthology.orchestra.run.smithi169.stdout:38654705670 2024-09-14T06:52:03.593 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-14T06:52:03.594 INFO:teuthology.orchestra.run.smithi169.stdout:38654705669 2024-09-14T06:52:03.594 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-14T06:52:03.928 INFO:teuthology.orchestra.run.smithi169.stdout:38654705668 2024-09-14T06:52:03.930 INFO:teuthology.orchestra.run.smithi169.stdout:38654705669 2024-09-14T06:52:03.939 INFO:teuthology.orchestra.run.smithi169.stdout:38654705668 2024-09-14T06:52:03.948 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.1 2024-09-14T06:52:03.952 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705669 for osd.2 2024-09-14T06:52:03.959 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.0 2024-09-14T06:52:04.949 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-14T06:52:04.952 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-14T06:52:04.960 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-14T06:52:05.276 INFO:teuthology.orchestra.run.smithi169.stdout:38654705670 2024-09-14T06:52:05.289 INFO:teuthology.orchestra.run.smithi169.stdout:38654705670 2024-09-14T06:52:05.293 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705670 for osd.1 2024-09-14T06:52:05.294 DEBUG:teuthology.parallel:result is None 2024-09-14T06:52:05.302 INFO:teuthology.orchestra.run.smithi169.stdout:38654705671 2024-09-14T06:52:05.307 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705670 for osd.0 2024-09-14T06:52:05.307 DEBUG:teuthology.parallel:result is None 2024-09-14T06:52:05.320 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705671 for osd.2 2024-09-14T06:52:05.320 DEBUG:teuthology.parallel:result is None 2024-09-14T06:52:05.320 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-14T06:52:05.320 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T06:52:05.635 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:05.636 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T06:52:05.652 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-14T06:52:04.916031+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":86836,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":85179,"kb_avail":283028684,"statfs":{"total":289910292480,"available":289821372416,"internally_reserved":0,"allocated":1613824,"data_stored":1302373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4784,"internal_metadata":87223632},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.877751"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.054336+0000","last_change":"2024-09-14T06:52:02.054336+0000","last_active":"2024-09-14T06:52:02.054336+0000","last_peered":"2024-09-14T06:52:02.054336+0000","last_clean":"2024-09-14T06:52:02.054336+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T06:52:02.054336+0000","last_undegraded":"2024-09-14T06:52:02.054336+0000","last_fullsized":"2024-09-14T06:52:02.054336+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:38:29.413964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.054459+0000","last_change":"2024-09-14T06:52:02.054459+0000","last_active":"2024-09-14T06:52:02.054459+0000","last_peered":"2024-09-14T06:52:02.054459+0000","last_clean":"2024-09-14T06:52:02.054459+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T06:52:02.054459+0000","last_undegraded":"2024-09-14T06:52:02.054459+0000","last_fullsized":"2024-09-14T06:52:02.054459+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:46:17.306187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.055138+0000","last_change":"2024-09-14T06:52:02.055138+0000","last_active":"2024-09-14T06:52:02.055138+0000","last_peered":"2024-09-14T06:52:02.055138+0000","last_clean":"2024-09-14T06:52:02.055138+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T06:52:02.055138+0000","last_undegraded":"2024-09-14T06:52:02.055138+0000","last_fullsized":"2024-09-14T06:52:02.055138+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:01:29.208041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.053552+0000","last_change":"2024-09-14T06:52:02.053552+0000","last_active":"2024-09-14T06:52:02.053552+0000","last_peered":"2024-09-14T06:52:02.053552+0000","last_clean":"2024-09-14T06:52:02.053552+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T06:52:02.053552+0000","last_undegraded":"2024-09-14T06:52:02.053552+0000","last_fullsized":"2024-09-14T06:52:02.053552+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:07.068171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.064642+0000","last_change":"2024-09-14T06:52:02.063567+0000","last_active":"2024-09-14T06:52:02.064642+0000","last_peered":"2024-09-14T06:52:02.064642+0000","last_clean":"2024-09-14T06:52:02.064642+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T06:52:02.064642+0000","last_undegraded":"2024-09-14T06:52:02.064642+0000","last_fullsized":"2024-09-14T06:52:02.064642+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:29.285798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.240204+0000","last_change":"2024-09-14T06:52:02.240204+0000","last_active":"2024-09-14T06:52:02.240204+0000","last_peered":"2024-09-14T06:52:02.240204+0000","last_clean":"2024-09-14T06:52:02.240204+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T06:52:02.240204+0000","last_undegraded":"2024-09-14T06:52:02.240204+0000","last_fullsized":"2024-09-14T06:52:02.240204+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:50.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.240272+0000","last_change":"2024-09-14T06:52:02.240272+0000","last_active":"2024-09-14T06:52:02.240272+0000","last_peered":"2024-09-14T06:52:02.240272+0000","last_clean":"2024-09-14T06:52:02.240272+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T06:52:02.240272+0000","last_undegraded":"2024-09-14T06:52:02.240272+0000","last_fullsized":"2024-09-14T06:52:02.240272+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:51:25.088755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.053793+0000","last_change":"2024-09-14T06:52:02.053793+0000","last_active":"2024-09-14T06:52:02.053793+0000","last_peered":"2024-09-14T06:52:02.053793+0000","last_clean":"2024-09-14T06:52:02.053793+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T06:52:02.053793+0000","last_undegraded":"2024-09-14T06:52:02.053793+0000","last_fullsized":"2024-09-14T06:52:02.053793+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:01:49.640848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":88,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.051967+0000","last_change":"2024-09-14T06:51:56.116608+0000","last_active":"2024-09-14T06:52:02.051967+0000","last_peered":"2024-09-14T06:52:02.051967+0000","last_clean":"2024-09-14T06:52:02.051967+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T06:52:02.051967+0000","last_undegraded":"2024-09-14T06:52:02.051967+0000","last_fullsized":"2024-09-14T06:52:02.051967+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_clean_scrub_stamp":"2024-09-14T06:51:55.005796+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:26.973731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705671,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":139264,"data_stored":40533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705670,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27756,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344084,"statfs":{"total":96636764160,"available":96608342016,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705670,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31912,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339928,"statfs":{"total":96636764160,"available":96604086272,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T06:52:05.653 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T06:52:05.936 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:05.936 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T06:52:05.952 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-14T06:52:04.916031+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":86836,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":85179,"kb_avail":283028684,"statfs":{"total":289910292480,"available":289821372416,"internally_reserved":0,"allocated":1613824,"data_stored":1302373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4784,"internal_metadata":87223632},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.877751"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.054336+0000","last_change":"2024-09-14T06:52:02.054336+0000","last_active":"2024-09-14T06:52:02.054336+0000","last_peered":"2024-09-14T06:52:02.054336+0000","last_clean":"2024-09-14T06:52:02.054336+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T06:52:02.054336+0000","last_undegraded":"2024-09-14T06:52:02.054336+0000","last_fullsized":"2024-09-14T06:52:02.054336+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:38:29.413964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.054459+0000","last_change":"2024-09-14T06:52:02.054459+0000","last_active":"2024-09-14T06:52:02.054459+0000","last_peered":"2024-09-14T06:52:02.054459+0000","last_clean":"2024-09-14T06:52:02.054459+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T06:52:02.054459+0000","last_undegraded":"2024-09-14T06:52:02.054459+0000","last_fullsized":"2024-09-14T06:52:02.054459+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:46:17.306187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.055138+0000","last_change":"2024-09-14T06:52:02.055138+0000","last_active":"2024-09-14T06:52:02.055138+0000","last_peered":"2024-09-14T06:52:02.055138+0000","last_clean":"2024-09-14T06:52:02.055138+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T06:52:02.055138+0000","last_undegraded":"2024-09-14T06:52:02.055138+0000","last_fullsized":"2024-09-14T06:52:02.055138+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:01:29.208041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.053552+0000","last_change":"2024-09-14T06:52:02.053552+0000","last_active":"2024-09-14T06:52:02.053552+0000","last_peered":"2024-09-14T06:52:02.053552+0000","last_clean":"2024-09-14T06:52:02.053552+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T06:52:02.053552+0000","last_undegraded":"2024-09-14T06:52:02.053552+0000","last_fullsized":"2024-09-14T06:52:02.053552+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:07.068171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.064642+0000","last_change":"2024-09-14T06:52:02.063567+0000","last_active":"2024-09-14T06:52:02.064642+0000","last_peered":"2024-09-14T06:52:02.064642+0000","last_clean":"2024-09-14T06:52:02.064642+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T06:52:02.064642+0000","last_undegraded":"2024-09-14T06:52:02.064642+0000","last_fullsized":"2024-09-14T06:52:02.064642+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:29.285798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.240204+0000","last_change":"2024-09-14T06:52:02.240204+0000","last_active":"2024-09-14T06:52:02.240204+0000","last_peered":"2024-09-14T06:52:02.240204+0000","last_clean":"2024-09-14T06:52:02.240204+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T06:52:02.240204+0000","last_undegraded":"2024-09-14T06:52:02.240204+0000","last_fullsized":"2024-09-14T06:52:02.240204+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:50.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.240272+0000","last_change":"2024-09-14T06:52:02.240272+0000","last_active":"2024-09-14T06:52:02.240272+0000","last_peered":"2024-09-14T06:52:02.240272+0000","last_clean":"2024-09-14T06:52:02.240272+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T06:52:02.240272+0000","last_undegraded":"2024-09-14T06:52:02.240272+0000","last_fullsized":"2024-09-14T06:52:02.240272+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:51:25.088755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.053793+0000","last_change":"2024-09-14T06:52:02.053793+0000","last_active":"2024-09-14T06:52:02.053793+0000","last_peered":"2024-09-14T06:52:02.053793+0000","last_clean":"2024-09-14T06:52:02.053793+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T06:52:02.053793+0000","last_undegraded":"2024-09-14T06:52:02.053793+0000","last_fullsized":"2024-09-14T06:52:02.053793+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:01:49.640848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":88,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-14T06:52:02.051967+0000","last_change":"2024-09-14T06:51:56.116608+0000","last_active":"2024-09-14T06:52:02.051967+0000","last_peered":"2024-09-14T06:52:02.051967+0000","last_clean":"2024-09-14T06:52:02.051967+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T06:52:02.051967+0000","last_undegraded":"2024-09-14T06:52:02.051967+0000","last_fullsized":"2024-09-14T06:52:02.051967+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_clean_scrub_stamp":"2024-09-14T06:51:55.005796+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:26.973731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705671,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":139264,"data_stored":40533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705670,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27756,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344084,"statfs":{"total":96636764160,"available":96608342016,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705670,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31912,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339928,"statfs":{"total":96636764160,"available":96604086272,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T06:52:05.953 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-14T06:52:05.953 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2024-09-14T06:52:05.953 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2024-09-14T06:52:05.953 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2024-09-14T06:52:06.269 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T06:52:06.270 INFO:teuthology.orchestra.run.smithi169.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T06:52:06.287 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2024-09-14T06:52:06.287 INFO:teuthology.run_tasks:Running task exec... 2024-09-14T06:52:06.297 INFO:teuthology.task.exec:Executing custom commands... 2024-09-14T06:52:06.298 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T06:52:06.298 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:06.665 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror] 2024-09-14T06:52:06.665 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBJMuVmspdYLxAAnqp+2Q9sPgKEnKSoWDGF6A== 2024-09-14T06:52:06.665 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-14T06:52:06.666 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:06.666 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror 2024-09-14T06:52:06.683 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:07.052 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.0] 2024-09-14T06:52:07.053 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBJMuVmCRmdNRAACsfW/PaTz+w2yuFQ482TeQ== 2024-09-14T06:52:07.053 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:07.053 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:07.053 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.0 2024-09-14T06:52:07.073 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:07.449 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.1] 2024-09-14T06:52:07.449 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBJMuVmOim6OhAAJEVCXOcISRe+StLwzA02ZA== 2024-09-14T06:52:07.450 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:07.450 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:07.450 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.1 2024-09-14T06:52:07.467 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:07.835 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.2] 2024-09-14T06:52:07.836 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBKMuVmU6cTBBAAUTbyGVf5p4Rmd3EkWgOsiA== 2024-09-14T06:52:07.836 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:07.836 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:07.836 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.2 2024-09-14T06:52:07.853 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:08.218 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.3] 2024-09-14T06:52:08.219 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBKMuVmI/AWCRAAw14wU7XMLA1o9oejUpAVGg== 2024-09-14T06:52:08.219 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:08.219 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:08.219 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.3 2024-09-14T06:52:08.237 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:08.591 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror] 2024-09-14T06:52:08.591 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBwMuVmfRo2OxAA94ze85NKZfFzClcFY+HDLA== 2024-09-14T06:52:08.591 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-14T06:52:08.591 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:08.592 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror 2024-09-14T06:52:08.610 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:08.971 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.0] 2024-09-14T06:52:08.971 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmTQDBBBAArr0BZHc25YiUHJfhNh8ySQ== 2024-09-14T06:52:08.971 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:08.972 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:08.972 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.0 2024-09-14T06:52:08.990 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:09.363 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.1] 2024-09-14T06:52:09.363 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVma5+/CRAAnMJyQbBZs6/0HcklUKwakg== 2024-09-14T06:52:09.363 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:09.363 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:09.363 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.1 2024-09-14T06:52:09.382 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:09.723 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.2] 2024-09-14T06:52:09.724 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmoantDhAAdkXZBMCvImQGjTdIpI2ZIA== 2024-09-14T06:52:09.724 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:09.724 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:09.724 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.2 2024-09-14T06:52:09.742 DEBUG:teuthology.orchestra.run.smithi169:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-14T06:52:10.104 INFO:teuthology.orchestra.run.smithi169.stdout:[client.mirror.3] 2024-09-14T06:52:10.104 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQBxMuVmDXz4ExAA7jLLw9omp2Y4EEz/ttKAvw== 2024-09-14T06:52:10.104 INFO:teuthology.orchestra.run.smithi169.stdout: caps mon = "profile rbd-mirror" 2024-09-14T06:52:10.104 INFO:teuthology.orchestra.run.smithi169.stdout: caps osd = "profile rbd" 2024-09-14T06:52:10.104 INFO:teuthology.orchestra.run.smithi169.stderr:updated caps for client.mirror.3 2024-09-14T06:52:10.123 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T06:52:10.134 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T06:52:10.137 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T06:52:10.137 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T06:52:10.179 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T06:52:10.181 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T06:52:10.181 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.228 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T06:52:10.228 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T06:52:10.230 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T06:52:10.231 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.336 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T06:52:10.336 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T06:52:10.339 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T06:52:10.339 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.387 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T06:52:10.387 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T06:52:10.389 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T06:52:10.389 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.438 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T06:52:10.439 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T06:52:10.441 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T06:52:10.441 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T06:52:10.488 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T06:52:10.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T06:52:10.490 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-14T06:52:10.538 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-14T06:52:10.538 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-14T06:52:10.540 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-14T06:52:10.540 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-09-14T06:52:10.589 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1726296730 2024-09-14T06:52:10.589 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-09-14T06:52:10.589 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-09-14T06:52:10.639 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1726296730 2024-09-14T06:52:10.639 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-09-14T06:52:10.639 INFO:teuthology.run_tasks:Running task workunit... 2024-09-14T06:52:10.692 INFO:tasks.workunit:Pulling workunits from ref 0d61375b8ab53be4c233f5b93e463edfdff9316a 2024-09-14T06:52:10.692 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T06:52:10.693 DEBUG:teuthology.orchestra.run.smithi169:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T06:52:10.693 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2024-09-14T06:52:10.693 INFO:tasks.rbd_mirror_thrash:waiting for 99 secs before thrashing 2024-09-14T06:52:10.693 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2024-09-14T06:52:10.694 INFO:tasks.rbd_mirror_thrash:waiting for 110 secs before thrashing 2024-09-14T06:52:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T06:52:10.699 INFO:teuthology.orchestra.run.smithi169.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2024-09-14T06:52:10.699 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T06:52:10.746 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T06:52:10.747 DEBUG:teuthology.orchestra.run.smithi169:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2024-09-14T06:52:10.798 INFO:tasks.workunit:timeout=6h 2024-09-14T06:52:10.798 INFO:tasks.workunit:cleanup=True 2024-09-14T06:52:10.798 DEBUG:teuthology.orchestra.run.smithi169:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout 0d61375b8ab53be4c233f5b93e463edfdff9316a 2024-09-14T06:52:10.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2024-09-14T06:53:03.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Note: switching to '0d61375b8ab53be4c233f5b93e463edfdff9316a'. 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:state without impacting any branches by switching back to a branch. 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: git switch -c 2024-09-14T06:53:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Or undo this operation with: 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: git switch - 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: 2024-09-14T06:53:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:HEAD is now at 0d61375b8ab Merge pull request #59485 from mcv21/cephadm_image_warn 2024-09-14T06:53:03.996 DEBUG:teuthology.orchestra.run.smithi169:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2024-09-14T06:53:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T06:53:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-14T06:53:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T06:53:04.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T06:53:04.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T06:53:04.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-14T06:53:04.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-14T06:53:04.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T06:53:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-14T06:53:05.042 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2024-09-14T06:53:05.042 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2024-09-14T06:53:05.090 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2024-09-14T06:53:05.091 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2024-09-14T06:53:05.091 DEBUG:teuthology.orchestra.run.smithi169:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0d61375b8ab53be4c233f5b93e463edfdff9316a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=pool RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-14T06:53:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ IMAGE_COUNT=50 2024-09-14T06:53:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ export LOCKDEP=0 2024-09-14T06:53:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ LOCKDEP=0 2024-09-14T06:53:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-14T06:53:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2024-09-14T06:53:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ RBD_MIRROR_INSTANCES=4 2024-09-14T06:53:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CLUSTER1=cluster1 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CLUSTER2=cluster2 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ PEER_CLUSTER_SUFFIX= 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ POOL=mirror 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ PARENT_POOL=mirror_parent 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ NS1=ns1 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ NS2=ns2 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ TEMPDIR= 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ID=mirror 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ RBD_MIRROR_MODE=journal 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ MIRROR_POOL_MODE=pool 2024-09-14T06:53:05.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' journal = snapshot ']' 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ export 'CEPH_ARGS=--id mirror' 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS='--id mirror' 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ LAST_MIRROR_INSTANCE=3 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_BIN=. 2024-09-14T06:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_SRC=. 2024-09-14T06:53:05.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' -e CMakeCache.txt ']' 2024-09-14T06:53:05.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 -gt 0 ']' 2024-09-14T06:53:05.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ setup 2024-09-14T06:53:05.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local c 2024-09-14T06:53:05.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ trap 'cleanup $?' INT TERM EXIT 2024-09-14T06:53:05.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ setup_tempdir 2024-09-14T06:53:05.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:05.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ mktemp -d 2024-09-14T06:53:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ TEMPDIR=/tmp/tmp.gBxNEDbwzY 2024-09-14T06:53:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z 1 ']' 2024-09-14T06:53:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ setup_pools cluster1 cluster2 2024-09-14T06:53:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local remote_cluster=cluster2 2024-09-14T06:53:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mon_map_file 2024-09-14T06:53:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mon_addr 2024-09-14T06:53:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local admin_key_file 2024-09-14T06:53:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local uuid 2024-09-14T06:53:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2024-09-14T06:53:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror' already exists 2024-09-14T06:53:06.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:06.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2024-09-14T06:53:07.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror_parent' already exists 2024-09-14T06:53:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:07.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 pool init mirror 2024-09-14T06:53:10.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:10.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2024-09-14T06:53:13.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:13.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:13.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2024-09-14T06:53:13.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2024-09-14T06:53:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2024-09-14T06:53:13.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2024-09-14T06:53:13.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2024-09-14T06:53:13.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2024-09-14T06:53:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z ']' 2024-09-14T06:53:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z ']' 2024-09-14T06:53:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_add cluster1 mirror cluster2 2024-09-14T06:53:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local client_cluster=cluster2 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local remote_cluster=cluster2 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local uuid_var_name 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local error_code 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local peer_uuid 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:53:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_uuid=a4da665d-d414-4f15-8a67-aa73c04a0235 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 -eq 0 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_add cluster1 mirror_parent cluster2 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror_parent 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local client_cluster=cluster2 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local remote_cluster=cluster2 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local uuid_var_name 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local error_code 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local peer_uuid 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:53:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2024-09-14T06:53:13.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_uuid=300772ae-3da4-49ea-9711-ae2cab4b257e 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 -eq 0 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ setup_pools cluster2 cluster1 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local remote_cluster=cluster1 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mon_map_file 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mon_addr 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local admin_key_file 2024-09-14T06:53:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local uuid 2024-09-14T06:53:13.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:13.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2024-09-14T06:53:14.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror' already exists 2024-09-14T06:53:14.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:14.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2024-09-14T06:53:15.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror_parent' already exists 2024-09-14T06:53:15.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:15.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 pool init mirror 2024-09-14T06:53:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2024-09-14T06:53:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T06:53:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2024-09-14T06:53:21.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2024-09-14T06:53:21.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2024-09-14T06:53:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2024-09-14T06:53:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2024-09-14T06:53:21.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2024-09-14T06:53:21.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z ']' 2024-09-14T06:53:21.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z ']' 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_add cluster2 mirror cluster1 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local client_cluster=cluster1 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local remote_cluster=cluster1 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local uuid_var_name 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local error_code 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local peer_uuid 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T06:53:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:53:21.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2024-09-14T06:53:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_uuid=38028a33-7525-4bcc-8aae-622389a81973 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 -eq 0 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_add cluster2 mirror_parent cluster1 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror_parent 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local client_cluster=cluster1 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local remote_cluster=cluster1 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local uuid_var_name 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local error_code 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local peer_uuid 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 16 32 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:53:21.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ peer_uuid=426a1671-f155-4154-898a-ae993d78ad2c 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 17 ']' 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 -eq 0 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirrors cluster1 2024-09-14T06:53:21.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:21.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:53:21.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster1:0 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster1:1 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T06:53:21.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster1:2 2024-09-14T06:53:21.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster1:3 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirrors cluster2 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster2:0 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:0 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster2:0 cluster instance 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster2:0 2024-09-14T06:53:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster2 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster2:0 ']' 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster2 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster2 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster2:1 2024-09-14T06:53:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:1 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster2:1 cluster instance 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster2:1 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster2 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster2:1 ']' 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster2 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster2 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster2:2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster2:2 cluster instance 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster2:2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster2:2 ']' 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster2 2024-09-14T06:53:21.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster2:3 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:3 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster2:3 cluster instance 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster2:3 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster2 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster2:3 ']' 2024-09-14T06:53:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster2 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster2 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T06:53:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T06:53:21.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 06:53:21 TEST: add image and test replay after client crashes 2024-09-14T06:53:21.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 06:53:21 TEST: add image and test replay after client crashes 2024-09-14T06:53:21.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=test 2024-09-14T06:53:21.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 512M 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror test 512M 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:53:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/test 2024-09-14T06:53:22.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T06:53:22.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:22.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:53:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T06:53:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T06:53:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T06:53:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T06:53:23.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:53:23.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:23.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:53:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:23.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:53:23.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:53:23.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:23.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:23.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:23.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:53:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T06:53:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T06:53:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T06:53:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T06:53:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T06:53:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T06:53:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:23.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T06:53:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T06:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T06:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T06:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T06:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T06:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:23.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:23.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:23.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T06:53:23.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T06:53:23.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T06:53:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T06:53:23.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:23.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:23.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:23.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:23.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T06:53:23.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T06:53:23.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T06:53:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T06:53:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T06:53:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T06:53:23.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:23.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:23.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:23.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T06:53:23.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:23.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:23.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:23.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:23.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 1 2024-09-14T06:53:24.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:53:24.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:53:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:53:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:53:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:53:24.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:24.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:53:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:53:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:53:24.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:24.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:24.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:53:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:24.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:24.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T06:53:24.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T06:53:24.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T06:53:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T06:53:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T06:53:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T06:53:24.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T06:53:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T06:53:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T06:53:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:24.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T06:53:24.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T06:53:24.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:24.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:24.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T06:53:24.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T06:53:24.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T06:53:24.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:24.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:24.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:24.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T06:53:24.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:24.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:24.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T06:53:24.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T06:53:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T06:53:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T06:53:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T06:53:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T06:53:25.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:25.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:25.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:25.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T06:53:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:25.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 2 2024-09-14T06:53:27.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:53:27.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:53:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:53:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:53:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:53:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:53:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:53:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:53:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:27.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:27.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T06:53:27.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T06:53:27.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:27.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:27.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T06:53:27.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:27.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T06:53:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T06:53:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T06:53:27.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:27.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:27.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:27.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T06:53:27.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:27.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T06:53:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:27.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T06:53:27.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T06:53:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T06:53:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T06:53:27.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:27.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:27.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:27.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:27.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T06:53:27.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:27.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:27.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T06:53:27.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T06:53:27.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T06:53:27.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:27.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:27.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:27.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T06:53:27.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:27.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:27.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:27.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:27.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 4 2024-09-14T06:53:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:53:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:53:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:53:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:53:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:53:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:53:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:53:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:53:31.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:53:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:31.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:31.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:31.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:31.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T06:53:31.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:31.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:31.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:31.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:31.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T06:53:31.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T06:53:31.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T06:53:31.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:31.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:31.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:31.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T06:53:31.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:31.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:31.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T06:53:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T06:53:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T06:53:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T06:53:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T06:53:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T06:53:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T06:53:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T06:53:32.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:32.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:32.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T06:53:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:32.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:32.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:32.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:32.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T06:53:40.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:53:40.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:40.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:40.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:40.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:40.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:53:40.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:40.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:53:40.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:53:40.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:53:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:40.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:40.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:40.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T06:53:40.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T06:53:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T06:53:40.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:40.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:40.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T06:53:40.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T06:53:40.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:40.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:40.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T06:53:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T06:53:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T06:53:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T06:53:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T06:53:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T06:53:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T06:53:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T06:53:40.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T06:53:40.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T06:53:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T06:53:40.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T06:53:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T06:53:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T06:53:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T06:53:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T06:53:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T06:53:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T06:53:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T06:53:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T06:53:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T06:53:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T06:53:40.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:40.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T06:53:48.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:53:48.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:53:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:53:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:53:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:53:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:53:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:53:48.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T06:53:48.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T06:53:48.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T06:53:48.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T06:53:48.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T06:53:48.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T06:53:48.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T06:53:48.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:53:48.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name= 2024-09-14T06:53:48.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 10 2024-09-14T06:53:48.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T06:53:48.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T06:53:48.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:53:48.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:48.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:48.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T06:53:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=11 2024-09-14T06:53:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:53:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T06:53:49.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2898670576076242) > weight (0.25) 2024-09-14T06:53:49.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6737891906502401) > weight (0.25) 2024-09-14T06:53:49.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9226942148827623) > weight (0.25) 2024-09-14T06:53:49.695 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-14T06:53:49.695 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-14T06:53:49.696 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-14T06:53:49.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T06:53:49.695+0000 7fa197d45640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 30478) UID: 1000 2024-09-14T06:53:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 62126080 bytes 2024-09-14T06:53:51.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 71408640 bytes 2024-09-14T06:53:53.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 157352960 bytes 2024-09-14T06:53:54.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 164511744 bytes 2024-09-14T06:53:54.697 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-14T06:53:54.697 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:53:54.697 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-14T06:53:54.697 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T06:53:54.697 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T06:53:54.700 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T06:53:54.700 INFO:tasks.rbd_mirror_thrash:waiting for 114 secs before thrashing 2024-09-14T06:53:56.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 218376192 bytes 2024-09-14T06:53:58.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:600 IOs, 284945408 bytes 2024-09-14T06:53:59.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:700 IOs, 318055424 bytes 2024-09-14T06:53:59.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T06:53:59.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:53:59.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T06:53:59.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:53:59.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T06:53:59.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:53:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:54:00.694 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.533367298292875) > weight (0.25) 2024-09-14T06:54:00.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.39156296244373723) > weight (0.25) 2024-09-14T06:54:00.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.44045217348776333) > weight (0.25) 2024-09-14T06:54:00.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5385110353248603) > weight (0.25) 2024-09-14T06:54:00.695 INFO:tasks.rbd_mirror_thrash:waiting for 73 secs before thrashing 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:54:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:53:44 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:53:42' 2024-09-14T06:54:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T06:54:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T06:54:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T06:54:01.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2024-09-14T06:54:01.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:54:01.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:54:01.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T06:54:01.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:54:01.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T06:54:01.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T06:54:01.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T06:54:01.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:53:44' 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 06:53:44' 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 06:53:44' 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:54:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:54:01.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T06:54:01.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK42144264mirror.1smithi16919.3.0-4775-g7f24e589trueOK42184267mirror.2smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42104261mirror.0smithi1692024-09-14 06:53:44cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 06:53:42' 2024-09-14T06:54:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T06:54:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T06:54:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:53:44' 2024-09-14T06:54:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T06:54:01.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T06:54:01.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:54:01.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T06:54:01.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap1 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap1 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1 2024-09-14T06:54:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2024-09-14T06:54:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T06:54:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:54:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T06:54:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T06:54:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T06:54:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T06:54:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T06:54:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T06:54:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:31.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:54:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:54:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:54:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:54:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:31.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:54:31.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:54:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T06:54:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T06:54:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T06:54:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T06:54:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T06:54:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T06:54:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T06:54:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:54:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:54:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:31.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:31.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:31.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:31.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=9656], [object_number=11, tag_tid=2, entry_tid=9655], [object_number=10, tag_tid=2, entry_tid=9654], [object_number=9, tag_tid=2, entry_tid=9653]]], state=connected] 2024-09-14T06:54:32.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:32.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=9656], [object_number=11, tag_tid=2, entry_tid=9655], [object_number=10, tag_tid=2, entry_tid=9654], [object_number=9, tag_tid=2, entry_tid=9653]]], state=connected] 2024-09-14T06:54:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=9656' '!=' '' 2024-09-14T06:54:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=9656' = '' 2024-09-14T06:54:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:32.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:32.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:32.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9656 2024-09-14T06:54:32.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 9656 2024-09-14T06:54:32.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:32.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:32.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:32.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:32.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:32.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:32.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:32.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:32.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:32.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:32.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:32.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:32.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:32.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:32.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:32.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:32.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:32.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:32.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:32.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:32.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:32.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:32.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:32.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:32.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:32.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=9770], [object_number=9, tag_tid=2, entry_tid=9769], [object_number=8, tag_tid=2, entry_tid=9768], [object_number=11, tag_tid=2, entry_tid=9767]]], state=connected] 2024-09-14T06:54:32.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:32.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:32.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=9770], [object_number=9, tag_tid=2, entry_tid=9769], [object_number=8, tag_tid=2, entry_tid=9768], [object_number=11, tag_tid=2, entry_tid=9767]]], state=connected] 2024-09-14T06:54:32.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=9770' '!=' 'object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=9770' = 'object_number=8, tag_tid=2, entry_tid=9656' 2024-09-14T06:54:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:32.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:32.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:32.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:32.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:32.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:32.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9770 2024-09-14T06:54:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 9770 2024-09-14T06:54:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:32.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:32.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:32.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:32.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:32.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:32.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:32.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:32.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:32.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:32.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:32.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:32.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=9896], [object_number=11, tag_tid=2, entry_tid=9895], [object_number=10, tag_tid=2, entry_tid=9894], [object_number=9, tag_tid=2, entry_tid=9893]]], state=connected] 2024-09-14T06:54:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:33.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:33.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:33.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:33.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=9896], [object_number=11, tag_tid=2, entry_tid=9895], [object_number=10, tag_tid=2, entry_tid=9894], [object_number=9, tag_tid=2, entry_tid=9893]]], state=connected] 2024-09-14T06:54:33.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=9896' '!=' 'object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=9896' = 'object_number=10, tag_tid=2, entry_tid=9770' 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9896 2024-09-14T06:54:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 9896 2024-09-14T06:54:33.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:33.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:33.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:33.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:33.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=10006], [object_number=9, tag_tid=2, entry_tid=10005], [object_number=8, tag_tid=2, entry_tid=10004], [object_number=11, tag_tid=2, entry_tid=10003]]], state=connected] 2024-09-14T06:54:33.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:33.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=10006], [object_number=9, tag_tid=2, entry_tid=10005], [object_number=8, tag_tid=2, entry_tid=10004], [object_number=11, tag_tid=2, entry_tid=10003]]], state=connected] 2024-09-14T06:54:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=10006' '!=' 'object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=10006' = 'object_number=8, tag_tid=2, entry_tid=9896' 2024-09-14T06:54:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:33.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:33.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:33.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:33.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:33.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:33.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:33.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:33.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:33.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10006 2024-09-14T06:54:33.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10006 2024-09-14T06:54:33.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:33.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:33.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:33.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:33.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:33.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:33.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:33.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:33.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:33.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:33.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:33.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:33.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=10117], [object_number=8, tag_tid=2, entry_tid=10116], [object_number=11, tag_tid=2, entry_tid=10115], [object_number=10, tag_tid=2, entry_tid=10114]]], state=connected] 2024-09-14T06:54:33.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:33.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:33.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=10117], [object_number=8, tag_tid=2, entry_tid=10116], [object_number=11, tag_tid=2, entry_tid=10115], [object_number=10, tag_tid=2, entry_tid=10114]]], state=connected] 2024-09-14T06:54:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=10117' '!=' 'object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=10117' = 'object_number=10, tag_tid=2, entry_tid=10006' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:34.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:34.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10117 2024-09-14T06:54:34.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10117 2024-09-14T06:54:34.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:34.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:34.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:34.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:34.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:34.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:34.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:34.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:34.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:34.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:34.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:34.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:34.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:34.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:34.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:34.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:34.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:34.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=10212], [object_number=11, tag_tid=2, entry_tid=10211], [object_number=10, tag_tid=2, entry_tid=10210], [object_number=9, tag_tid=2, entry_tid=10209]]], state=connected] 2024-09-14T06:54:34.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:34.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:34.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=10212], [object_number=11, tag_tid=2, entry_tid=10211], [object_number=10, tag_tid=2, entry_tid=10210], [object_number=9, tag_tid=2, entry_tid=10209]]], state=connected] 2024-09-14T06:54:34.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=10212' '!=' 'object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=10212' = 'object_number=9, tag_tid=2, entry_tid=10117' 2024-09-14T06:54:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:34.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:34.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:34.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:34.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:34.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:34.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:34.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:34.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10212 2024-09-14T06:54:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10212 2024-09-14T06:54:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:34.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:34.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:34.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:34.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:34.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:34.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:34.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:34.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:34.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:34.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:34.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:34.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:34.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:34.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:34.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:34.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:34.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:34.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:34.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:34.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:34.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=10330], [object_number=9, tag_tid=2, entry_tid=10329], [object_number=8, tag_tid=2, entry_tid=10328], [object_number=11, tag_tid=2, entry_tid=10327]]], state=connected] 2024-09-14T06:54:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:34.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:34.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:34.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:34.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:34.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:34.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:35.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:35.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:35.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:35.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=10330], [object_number=9, tag_tid=2, entry_tid=10329], [object_number=8, tag_tid=2, entry_tid=10328], [object_number=11, tag_tid=2, entry_tid=10327]]], state=connected] 2024-09-14T06:54:35.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=10330' '!=' 'object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=10330' = 'object_number=8, tag_tid=2, entry_tid=10212' 2024-09-14T06:54:35.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:35.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:35.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:35.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:35.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:35.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:35.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10330 2024-09-14T06:54:35.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10330 2024-09-14T06:54:35.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:35.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:35.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:35.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:35.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:35.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:35.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:35.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:35.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:35.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:35.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:35.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:35.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:35.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:35.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:35.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:35.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:35.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:35.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=10456], [object_number=11, tag_tid=2, entry_tid=10455], [object_number=10, tag_tid=2, entry_tid=10454], [object_number=9, tag_tid=2, entry_tid=10453]]], state=connected] 2024-09-14T06:54:35.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=10456], [object_number=11, tag_tid=2, entry_tid=10455], [object_number=10, tag_tid=2, entry_tid=10454], [object_number=9, tag_tid=2, entry_tid=10453]]], state=connected] 2024-09-14T06:54:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=10456' '!=' 'object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=10456' = 'object_number=10, tag_tid=2, entry_tid=10330' 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:35.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:35.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:35.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:35.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:35.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10456 2024-09-14T06:54:35.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10456 2024-09-14T06:54:35.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:35.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:35.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:35.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:35.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:35.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:35.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:35.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:35.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=10581], [object_number=8, tag_tid=2, entry_tid=10580], [object_number=11, tag_tid=2, entry_tid=10579], [object_number=10, tag_tid=2, entry_tid=10578]]], state=connected] 2024-09-14T06:54:35.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=10581], [object_number=8, tag_tid=2, entry_tid=10580], [object_number=11, tag_tid=2, entry_tid=10579], [object_number=10, tag_tid=2, entry_tid=10578]]], state=connected] 2024-09-14T06:54:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=10581' '!=' 'object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=10581' = 'object_number=8, tag_tid=2, entry_tid=10456' 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:35.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10581 2024-09-14T06:54:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10581 2024-09-14T06:54:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:36.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:36.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:36.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:36.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:36.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:36.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:36.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:36.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:36.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:36.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:36.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:36.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:36.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:36.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:36.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:36.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:36.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=10696], [object_number=11, tag_tid=2, entry_tid=10695], [object_number=10, tag_tid=2, entry_tid=10694], [object_number=9, tag_tid=2, entry_tid=10693]]], state=connected] 2024-09-14T06:54:36.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:36.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:36.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:36.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:36.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=10696], [object_number=11, tag_tid=2, entry_tid=10695], [object_number=10, tag_tid=2, entry_tid=10694], [object_number=9, tag_tid=2, entry_tid=10693]]], state=connected] 2024-09-14T06:54:36.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=10696' '!=' 'object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=10696' = 'object_number=9, tag_tid=2, entry_tid=10581' 2024-09-14T06:54:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10696 2024-09-14T06:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10696 2024-09-14T06:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:36.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:36.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:36.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:36.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:36.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:36.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:36.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=10819], [object_number=10, tag_tid=2, entry_tid=10818], [object_number=9, tag_tid=2, entry_tid=10817], [object_number=8, tag_tid=2, entry_tid=10816]]], state=connected] 2024-09-14T06:54:36.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:36.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:36.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:36.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=10819], [object_number=10, tag_tid=2, entry_tid=10818], [object_number=9, tag_tid=2, entry_tid=10817], [object_number=8, tag_tid=2, entry_tid=10816]]], state=connected] 2024-09-14T06:54:36.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=10819' '!=' 'object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=10819' = 'object_number=8, tag_tid=2, entry_tid=10696' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:36.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10819 2024-09-14T06:54:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10819 2024-09-14T06:54:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:36.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:36.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:37.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:37.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:37.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:37.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:37.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:37.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:37.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:37.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:37.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:37.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:37.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:37.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:37.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:37.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:37.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:37.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:37.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=10943], [object_number=10, tag_tid=2, entry_tid=10942], [object_number=9, tag_tid=2, entry_tid=10941], [object_number=8, tag_tid=2, entry_tid=10940]]], state=connected] 2024-09-14T06:54:37.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:37.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:37.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:37.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:37.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:37.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:37.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:37.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:37.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=10943], [object_number=10, tag_tid=2, entry_tid=10942], [object_number=9, tag_tid=2, entry_tid=10941], [object_number=8, tag_tid=2, entry_tid=10940]]], state=connected] 2024-09-14T06:54:37.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=10943' '!=' 'object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=10943' = 'object_number=11, tag_tid=2, entry_tid=10819' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10943 2024-09-14T06:54:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 10943 2024-09-14T06:54:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:37.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:37.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:37.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:37.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:37.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:37.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:37.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:37.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:37.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:37.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:37.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:37.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:37.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:37.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:37.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:37.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:37.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:37.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:37.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=11065], [object_number=8, tag_tid=2, entry_tid=11064], [object_number=11, tag_tid=2, entry_tid=11063], [object_number=10, tag_tid=2, entry_tid=11062]]], state=connected] 2024-09-14T06:54:37.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=11065], [object_number=8, tag_tid=2, entry_tid=11064], [object_number=11, tag_tid=2, entry_tid=11063], [object_number=10, tag_tid=2, entry_tid=11062]]], state=connected] 2024-09-14T06:54:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=11065' '!=' 'object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=11065' = 'object_number=11, tag_tid=2, entry_tid=10943' 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:37.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:37.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:37.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:37.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:37.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:37.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:37.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11065 2024-09-14T06:54:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11065 2024-09-14T06:54:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:38.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:38.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:38.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:38.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:38.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:38.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=11184], [object_number=11, tag_tid=2, entry_tid=11183], [object_number=10, tag_tid=2, entry_tid=11182], [object_number=9, tag_tid=2, entry_tid=11181]]], state=connected] 2024-09-14T06:54:38.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:38.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=11184], [object_number=11, tag_tid=2, entry_tid=11183], [object_number=10, tag_tid=2, entry_tid=11182], [object_number=9, tag_tid=2, entry_tid=11181]]], state=connected] 2024-09-14T06:54:38.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=11184' '!=' 'object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=11184' = 'object_number=9, tag_tid=2, entry_tid=11065' 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:38.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:38.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:38.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:38.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:38.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:38.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:38.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:38.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:38.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11184 2024-09-14T06:54:38.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11184 2024-09-14T06:54:38.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:38.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:38.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:38.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:38.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:38.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:38.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:38.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:38.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:38.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:38.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:38.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:38.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:38.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:38.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:38.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=11309], [object_number=8, tag_tid=2, entry_tid=11308], [object_number=11, tag_tid=2, entry_tid=11307], [object_number=10, tag_tid=2, entry_tid=11306]]], state=connected] 2024-09-14T06:54:38.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:38.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:38.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:38.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:38.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:38.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=11309], [object_number=8, tag_tid=2, entry_tid=11308], [object_number=11, tag_tid=2, entry_tid=11307], [object_number=10, tag_tid=2, entry_tid=11306]]], state=connected] 2024-09-14T06:54:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:38.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:38.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=11309' '!=' 'object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=11309' = 'object_number=8, tag_tid=2, entry_tid=11184' 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:38.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:38.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:38.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11309 2024-09-14T06:54:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11309 2024-09-14T06:54:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:39.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:39.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:39.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:39.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:39.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:39.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:39.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:39.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:39.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:39.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:39.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:39.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:39.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:39.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:39.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=11433], [object_number=8, tag_tid=2, entry_tid=11432], [object_number=11, tag_tid=2, entry_tid=11431], [object_number=10, tag_tid=2, entry_tid=11430]]], state=connected] 2024-09-14T06:54:39.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:39.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:39.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:39.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=11433], [object_number=8, tag_tid=2, entry_tid=11432], [object_number=11, tag_tid=2, entry_tid=11431], [object_number=10, tag_tid=2, entry_tid=11430]]], state=connected] 2024-09-14T06:54:39.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=11433' '!=' 'object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=2, entry_tid=11433' = 'object_number=9, tag_tid=2, entry_tid=11309' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:39.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:39.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:39.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:39.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:39.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:39.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:39.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11433 2024-09-14T06:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11433 2024-09-14T06:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:39.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:39.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:39.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:39.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:39.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:39.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:39.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:39.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:39.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:39.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:39.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:39.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:39.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:39.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=11547], [object_number=10, tag_tid=2, entry_tid=11546], [object_number=9, tag_tid=2, entry_tid=11545], [object_number=8, tag_tid=2, entry_tid=11544]]], state=connected] 2024-09-14T06:54:39.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:39.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:39.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:39.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:39.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:39.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:39.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=11547], [object_number=10, tag_tid=2, entry_tid=11546], [object_number=9, tag_tid=2, entry_tid=11545], [object_number=8, tag_tid=2, entry_tid=11544]]], state=connected] 2024-09-14T06:54:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=11547' '!=' 'object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=11547' = 'object_number=9, tag_tid=2, entry_tid=11433' 2024-09-14T06:54:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:39.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:39.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:39.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:39.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:39.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:39.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:39.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11547 2024-09-14T06:54:39.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11547 2024-09-14T06:54:39.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:39.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:39.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:40.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:40.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:40.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:40.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:40.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:40.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:40.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:40.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:40.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:40.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:40.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:40.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:40.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:40.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:40.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:40.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:40.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:40.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:40.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=11670], [object_number=9, tag_tid=2, entry_tid=11669], [object_number=8, tag_tid=2, entry_tid=11668], [object_number=11, tag_tid=2, entry_tid=11667]]], state=connected] 2024-09-14T06:54:40.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:40.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:40.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:40.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:40.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=11670], [object_number=9, tag_tid=2, entry_tid=11669], [object_number=8, tag_tid=2, entry_tid=11668], [object_number=11, tag_tid=2, entry_tid=11667]]], state=connected] 2024-09-14T06:54:40.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=11670' '!=' 'object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=11670' = 'object_number=11, tag_tid=2, entry_tid=11547' 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11670 2024-09-14T06:54:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11670 2024-09-14T06:54:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:40.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:40.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:40.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:40.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:40.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:40.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:40.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:40.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:40.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:40.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:40.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=11819], [object_number=10, tag_tid=2, entry_tid=11818], [object_number=9, tag_tid=2, entry_tid=11817], [object_number=8, tag_tid=2, entry_tid=11816]]], state=connected] 2024-09-14T06:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:40.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:40.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=11819], [object_number=10, tag_tid=2, entry_tid=11818], [object_number=9, tag_tid=2, entry_tid=11817], [object_number=8, tag_tid=2, entry_tid=11816]]], state=connected] 2024-09-14T06:54:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=11819' '!=' 'object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=11819' = 'object_number=10, tag_tid=2, entry_tid=11670' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:40.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:40.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:40.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:40.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:40.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:40.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:40.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:40.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:40.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:40.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11819 2024-09-14T06:54:40.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11819 2024-09-14T06:54:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:41.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:41.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:41.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:41.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:41.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:41.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:41.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:41.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:41.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:41.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:41.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:41.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:41.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:41.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=11947], [object_number=10, tag_tid=2, entry_tid=11946], [object_number=9, tag_tid=2, entry_tid=11945], [object_number=8, tag_tid=2, entry_tid=11944]]], state=connected] 2024-09-14T06:54:41.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:41.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:41.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:41.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:41.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:41.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=2, entry_tid=11947], [object_number=10, tag_tid=2, entry_tid=11946], [object_number=9, tag_tid=2, entry_tid=11945], [object_number=8, tag_tid=2, entry_tid=11944]]], state=connected] 2024-09-14T06:54:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=11947' '!=' 'object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=2, entry_tid=11947' = 'object_number=11, tag_tid=2, entry_tid=11819' 2024-09-14T06:54:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:41.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:41.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:41.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11947 2024-09-14T06:54:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 11947 2024-09-14T06:54:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:41.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:41.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:41.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:41.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:41.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:41.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:41.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:41.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:41.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:41.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=12064], [object_number=11, tag_tid=2, entry_tid=12063], [object_number=10, tag_tid=2, entry_tid=12062], [object_number=9, tag_tid=2, entry_tid=12061]]], state=connected] 2024-09-14T06:54:41.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:41.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:41.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:41.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:41.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:41.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:41.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:41.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:42.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:42.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:42.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:42.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=2, entry_tid=12064], [object_number=11, tag_tid=2, entry_tid=12063], [object_number=10, tag_tid=2, entry_tid=12062], [object_number=9, tag_tid=2, entry_tid=12061]]], state=connected] 2024-09-14T06:54:42.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=12064' '!=' 'object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=2, entry_tid=12064' = 'object_number=11, tag_tid=2, entry_tid=11947' 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:42.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:42.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:42.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:42.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12064 2024-09-14T06:54:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12064 2024-09-14T06:54:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:42.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:42.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:42.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:42.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:42.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:42.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:42.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:42.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:42.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:42.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:42.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:42.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=12242], [object_number=9, tag_tid=2, entry_tid=12241], [object_number=8, tag_tid=2, entry_tid=12240], [object_number=11, tag_tid=2, entry_tid=12239]]], state=connected] 2024-09-14T06:54:42.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:42.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:42.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:42.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:42.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:42.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:42.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:42.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=12242], [object_number=9, tag_tid=2, entry_tid=12241], [object_number=8, tag_tid=2, entry_tid=12240], [object_number=11, tag_tid=2, entry_tid=12239]]], state=connected] 2024-09-14T06:54:42.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=12242' '!=' 'object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=12242' = 'object_number=8, tag_tid=2, entry_tid=12064' 2024-09-14T06:54:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12242 2024-09-14T06:54:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12242 2024-09-14T06:54:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:43.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:43.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:43.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:43.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:43.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:43.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:43.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:43.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:43.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:43.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:43.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:43.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:43.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:43.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:43.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:43.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:43.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:43.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:43.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:43.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=12474], [object_number=9, tag_tid=2, entry_tid=12473], [object_number=8, tag_tid=2, entry_tid=12472], [object_number=11, tag_tid=2, entry_tid=12471]]], state=connected] 2024-09-14T06:54:43.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:43.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:43.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:43.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:43.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:43.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:43.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:43.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=2, entry_tid=12474], [object_number=9, tag_tid=2, entry_tid=12473], [object_number=8, tag_tid=2, entry_tid=12472], [object_number=11, tag_tid=2, entry_tid=12471]]], state=connected] 2024-09-14T06:54:43.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=12474' '!=' 'object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=2, entry_tid=12474' = 'object_number=10, tag_tid=2, entry_tid=12242' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:43.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:43.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:43.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:43.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:43.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:43.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:43.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:43.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12474 2024-09-14T06:54:43.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12474 2024-09-14T06:54:43.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:43.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:43.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:43.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:43.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:43.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:43.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:43.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=12552], [object_number=15, tag_tid=2, entry_tid=12551], [object_number=14, tag_tid=2, entry_tid=12550], [object_number=9, tag_tid=2, entry_tid=12549]]], state=connected] 2024-09-14T06:54:43.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:43.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T06:54:43.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:43.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:43.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:43.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=12552], [object_number=15, tag_tid=2, entry_tid=12551], [object_number=14, tag_tid=2, entry_tid=12550], [object_number=9, tag_tid=2, entry_tid=12549]]], state=connected] 2024-09-14T06:54:43.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:43.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:43.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:43.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=12552' '!=' 'object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=12552' = 'object_number=10, tag_tid=2, entry_tid=12474' 2024-09-14T06:54:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:43.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:43.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:43.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12552 2024-09-14T06:54:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12552 2024-09-14T06:54:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:44.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:44.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=12669], [object_number=12, tag_tid=2, entry_tid=12668], [object_number=15, tag_tid=2, entry_tid=12667], [object_number=14, tag_tid=2, entry_tid=12666]]], state=connected] 2024-09-14T06:54:44.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:44.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:44.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:44.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=12669], [object_number=12, tag_tid=2, entry_tid=12668], [object_number=15, tag_tid=2, entry_tid=12667], [object_number=14, tag_tid=2, entry_tid=12666]]], state=connected] 2024-09-14T06:54:44.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=12669' '!=' 'object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=12669' = 'object_number=12, tag_tid=2, entry_tid=12552' 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:44.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:44.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:44.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:44.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:44.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:44.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:44.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12669 2024-09-14T06:54:44.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12669 2024-09-14T06:54:44.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:44.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:44.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:44.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:44.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:44.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:44.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:44.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=12794], [object_number=13, tag_tid=2, entry_tid=12793], [object_number=12, tag_tid=2, entry_tid=12792], [object_number=15, tag_tid=2, entry_tid=12791]]], state=connected] 2024-09-14T06:54:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:44.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:44.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:44.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:44.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=12794], [object_number=13, tag_tid=2, entry_tid=12793], [object_number=12, tag_tid=2, entry_tid=12792], [object_number=15, tag_tid=2, entry_tid=12791]]], state=connected] 2024-09-14T06:54:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=12794' '!=' 'object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=12794' = 'object_number=13, tag_tid=2, entry_tid=12669' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:44.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:44.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:44.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:44.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12794 2024-09-14T06:54:44.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12794 2024-09-14T06:54:44.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:44.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:44.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:45.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:45.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:45.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:45.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:45.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:45.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:45.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:45.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:45.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:45.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:45.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:45.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:45.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:45.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:45.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=12916], [object_number=15, tag_tid=2, entry_tid=12915], [object_number=14, tag_tid=2, entry_tid=12914], [object_number=13, tag_tid=2, entry_tid=12913]]], state=connected] 2024-09-14T06:54:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:45.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:45.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:45.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:45.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=12916], [object_number=15, tag_tid=2, entry_tid=12915], [object_number=14, tag_tid=2, entry_tid=12914], [object_number=13, tag_tid=2, entry_tid=12913]]], state=connected] 2024-09-14T06:54:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:45.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=12916' '!=' 'object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=12916' = 'object_number=14, tag_tid=2, entry_tid=12794' 2024-09-14T06:54:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:45.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:45.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:45.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:45.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:45.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:45.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:45.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12916 2024-09-14T06:54:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 12916 2024-09-14T06:54:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:45.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:45.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:45.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:45.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:45.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:45.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:45.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:45.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:45.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:45.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:45.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:45.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:45.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13090], [object_number=13, tag_tid=2, entry_tid=13089], [object_number=12, tag_tid=2, entry_tid=13088], [object_number=15, tag_tid=2, entry_tid=13087]]], state=connected] 2024-09-14T06:54:45.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:46.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:46.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:46.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:46.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13090], [object_number=13, tag_tid=2, entry_tid=13089], [object_number=12, tag_tid=2, entry_tid=13088], [object_number=15, tag_tid=2, entry_tid=13087]]], state=connected] 2024-09-14T06:54:46.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13090' '!=' 'object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13090' = 'object_number=12, tag_tid=2, entry_tid=12916' 2024-09-14T06:54:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:46.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:46.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:46.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:46.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:46.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13090 2024-09-14T06:54:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13090 2024-09-14T06:54:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:46.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:46.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:46.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:46.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:46.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:46.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:46.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:46.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:46.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:46.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:46.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=13215], [object_number=14, tag_tid=2, entry_tid=13214], [object_number=13, tag_tid=2, entry_tid=13213], [object_number=12, tag_tid=2, entry_tid=13212]]], state=connected] 2024-09-14T06:54:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:46.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:46.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:46.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=13215], [object_number=14, tag_tid=2, entry_tid=13214], [object_number=13, tag_tid=2, entry_tid=13213], [object_number=12, tag_tid=2, entry_tid=13212]]], state=connected] 2024-09-14T06:54:46.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=13215' '!=' 'object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=13215' = 'object_number=14, tag_tid=2, entry_tid=13090' 2024-09-14T06:54:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:46.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:46.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13215 2024-09-14T06:54:46.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13215 2024-09-14T06:54:46.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:46.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:46.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:46.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:46.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:46.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:46.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:46.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:46.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:46.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:46.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:46.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:46.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:46.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:46.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:46.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:46.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:46.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:46.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:46.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:46.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:46.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:46.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:46.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:46.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=13339], [object_number=14, tag_tid=2, entry_tid=13338], [object_number=13, tag_tid=2, entry_tid=13337], [object_number=12, tag_tid=2, entry_tid=13336]]], state=connected] 2024-09-14T06:54:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:47.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:47.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=13339], [object_number=14, tag_tid=2, entry_tid=13338], [object_number=13, tag_tid=2, entry_tid=13337], [object_number=12, tag_tid=2, entry_tid=13336]]], state=connected] 2024-09-14T06:54:47.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=13339' '!=' 'object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:47.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:47.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=13339' = 'object_number=15, tag_tid=2, entry_tid=13215' 2024-09-14T06:54:47.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:47.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:47.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:47.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13339 2024-09-14T06:54:47.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13339 2024-09-14T06:54:47.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:47.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:47.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:47.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:47.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:47.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:47.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:47.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:47.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:47.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:47.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:47.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:47.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:47.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13462], [object_number=13, tag_tid=2, entry_tid=13461], [object_number=12, tag_tid=2, entry_tid=13460], [object_number=15, tag_tid=2, entry_tid=13459]]], state=connected] 2024-09-14T06:54:47.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:47.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:47.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:47.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:47.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:47.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:47.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13462], [object_number=13, tag_tid=2, entry_tid=13461], [object_number=12, tag_tid=2, entry_tid=13460], [object_number=15, tag_tid=2, entry_tid=13459]]], state=connected] 2024-09-14T06:54:47.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13462' '!=' 'object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13462' = 'object_number=15, tag_tid=2, entry_tid=13339' 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:47.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:47.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:47.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:47.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:47.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:47.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13462 2024-09-14T06:54:47.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13462 2024-09-14T06:54:47.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:47.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:47.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:47.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:47.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:47.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:47.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:47.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:47.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:47.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:47.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:47.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:47.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:47.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:47.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:47.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:47.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:47.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:47.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:47.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:47.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13586], [object_number=13, tag_tid=2, entry_tid=13585], [object_number=12, tag_tid=2, entry_tid=13584], [object_number=15, tag_tid=2, entry_tid=13583]]], state=connected] 2024-09-14T06:54:47.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:47.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:47.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:47.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:47.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:47.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13586], [object_number=13, tag_tid=2, entry_tid=13585], [object_number=12, tag_tid=2, entry_tid=13584], [object_number=15, tag_tid=2, entry_tid=13583]]], state=connected] 2024-09-14T06:54:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:47.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13586' '!=' 'object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13586' = 'object_number=14, tag_tid=2, entry_tid=13462' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:47.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:47.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:47.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13586 2024-09-14T06:54:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13586 2024-09-14T06:54:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:48.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:48.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:48.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:48.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:48.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:48.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:48.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:48.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:48.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:48.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:48.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:48.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13714], [object_number=13, tag_tid=2, entry_tid=13713], [object_number=12, tag_tid=2, entry_tid=13712], [object_number=15, tag_tid=2, entry_tid=13711]]], state=connected] 2024-09-14T06:54:48.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:48.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:48.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:48.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:48.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:48.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:48.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:48.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:48.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13714], [object_number=13, tag_tid=2, entry_tid=13713], [object_number=12, tag_tid=2, entry_tid=13712], [object_number=15, tag_tid=2, entry_tid=13711]]], state=connected] 2024-09-14T06:54:48.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13714' '!=' 'object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13714' = 'object_number=14, tag_tid=2, entry_tid=13586' 2024-09-14T06:54:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:48.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:48.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:48.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:48.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:48.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:48.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13714 2024-09-14T06:54:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13714 2024-09-14T06:54:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:48.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:48.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:48.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:48.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:48.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:48.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:48.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:48.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:48.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:48.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:48.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:48.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:48.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:48.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:48.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:48.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:48.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:48.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:48.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:48.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:48.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:48.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13826], [object_number=13, tag_tid=2, entry_tid=13825], [object_number=12, tag_tid=2, entry_tid=13824], [object_number=15, tag_tid=2, entry_tid=13823]]], state=connected] 2024-09-14T06:54:48.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13826], [object_number=13, tag_tid=2, entry_tid=13825], [object_number=12, tag_tid=2, entry_tid=13824], [object_number=15, tag_tid=2, entry_tid=13823]]], state=connected] 2024-09-14T06:54:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13826' '!=' 'object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13826' = 'object_number=14, tag_tid=2, entry_tid=13714' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:48.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:48.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:48.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:48.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:48.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:48.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:48.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:48.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13826 2024-09-14T06:54:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13826 2024-09-14T06:54:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:49.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:49.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:49.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:49.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:49.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:49.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:49.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:49.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:49.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:49.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:49.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:49.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13934], [object_number=13, tag_tid=2, entry_tid=13933], [object_number=12, tag_tid=2, entry_tid=13932], [object_number=15, tag_tid=2, entry_tid=13931]]], state=connected] 2024-09-14T06:54:49.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:49.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:49.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=13934], [object_number=13, tag_tid=2, entry_tid=13933], [object_number=12, tag_tid=2, entry_tid=13932], [object_number=15, tag_tid=2, entry_tid=13931]]], state=connected] 2024-09-14T06:54:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13934' '!=' 'object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:49.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=13934' = 'object_number=14, tag_tid=2, entry_tid=13826' 2024-09-14T06:54:49.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:49.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:49.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:49.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:49.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:49.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:49.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:49.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:49.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:49.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13934 2024-09-14T06:54:49.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 13934 2024-09-14T06:54:49.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:49.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:49.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:49.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:49.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:49.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:49.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:49.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:49.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:49.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:49.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:49.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14063], [object_number=14, tag_tid=2, entry_tid=14062], [object_number=13, tag_tid=2, entry_tid=14061], [object_number=12, tag_tid=2, entry_tid=14060]]], state=connected] 2024-09-14T06:54:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:49.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:49.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:49.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:49.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:49.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14063], [object_number=14, tag_tid=2, entry_tid=14062], [object_number=13, tag_tid=2, entry_tid=14061], [object_number=12, tag_tid=2, entry_tid=14060]]], state=connected] 2024-09-14T06:54:49.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14063' '!=' 'object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14063' = 'object_number=14, tag_tid=2, entry_tid=13934' 2024-09-14T06:54:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:49.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:49.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:49.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:49.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:49.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:49.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:49.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:49.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:49.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:49.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14063 2024-09-14T06:54:49.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14063 2024-09-14T06:54:49.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:49.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:49.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:50.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:50.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:50.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:50.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:50.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:50.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:50.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:50.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:50.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:50.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:50.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14175], [object_number=14, tag_tid=2, entry_tid=14174], [object_number=13, tag_tid=2, entry_tid=14173], [object_number=12, tag_tid=2, entry_tid=14172]]], state=connected] 2024-09-14T06:54:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:50.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:50.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:50.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:50.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14175], [object_number=14, tag_tid=2, entry_tid=14174], [object_number=13, tag_tid=2, entry_tid=14173], [object_number=12, tag_tid=2, entry_tid=14172]]], state=connected] 2024-09-14T06:54:50.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14175' '!=' 'object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14175' = 'object_number=15, tag_tid=2, entry_tid=14063' 2024-09-14T06:54:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:50.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:50.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:50.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:50.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:50.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14175 2024-09-14T06:54:50.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14175 2024-09-14T06:54:50.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:50.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:50.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:50.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:50.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:50.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:50.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:50.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:50.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=14338], [object_number=13, tag_tid=2, entry_tid=14337], [object_number=12, tag_tid=2, entry_tid=14336], [object_number=15, tag_tid=2, entry_tid=14335]]], state=connected] 2024-09-14T06:54:50.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:50.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=14338], [object_number=13, tag_tid=2, entry_tid=14337], [object_number=12, tag_tid=2, entry_tid=14336], [object_number=15, tag_tid=2, entry_tid=14335]]], state=connected] 2024-09-14T06:54:50.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:50.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:50.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=14338' '!=' 'object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=14338' = 'object_number=15, tag_tid=2, entry_tid=14175' 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:50.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:50.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:50.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:50.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:50.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:50.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:50.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14338 2024-09-14T06:54:50.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14338 2024-09-14T06:54:50.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:50.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:50.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:51.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:51.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:51.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:51.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:51.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:51.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:51.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14471], [object_number=14, tag_tid=2, entry_tid=14470], [object_number=13, tag_tid=2, entry_tid=14469], [object_number=12, tag_tid=2, entry_tid=14468]]], state=connected] 2024-09-14T06:54:51.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:51.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:51.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14471], [object_number=14, tag_tid=2, entry_tid=14470], [object_number=13, tag_tid=2, entry_tid=14469], [object_number=12, tag_tid=2, entry_tid=14468]]], state=connected] 2024-09-14T06:54:51.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14471' '!=' 'object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14471' = 'object_number=14, tag_tid=2, entry_tid=14338' 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:51.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:51.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:51.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14471 2024-09-14T06:54:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14471 2024-09-14T06:54:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:51.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:51.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:51.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:51.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:51.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:51.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:51.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:51.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:51.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:51.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:51.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:51.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:51.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=14593], [object_number=12, tag_tid=2, entry_tid=14592], [object_number=15, tag_tid=2, entry_tid=14591], [object_number=14, tag_tid=2, entry_tid=14590]]], state=connected] 2024-09-14T06:54:51.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:51.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:51.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:51.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=14593], [object_number=12, tag_tid=2, entry_tid=14592], [object_number=15, tag_tid=2, entry_tid=14591], [object_number=14, tag_tid=2, entry_tid=14590]]], state=connected] 2024-09-14T06:54:51.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:51.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:51.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:51.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=14593' '!=' 'object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=14593' = 'object_number=15, tag_tid=2, entry_tid=14471' 2024-09-14T06:54:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:51.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:51.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:51.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:51.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:51.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:51.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14593 2024-09-14T06:54:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14593 2024-09-14T06:54:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:52.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:52.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:52.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:52.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:52.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:52.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:52.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:52.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:52.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:52.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:52.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:52.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:52.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:52.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:52.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=14718], [object_number=13, tag_tid=2, entry_tid=14717], [object_number=12, tag_tid=2, entry_tid=14716], [object_number=15, tag_tid=2, entry_tid=14715]]], state=connected] 2024-09-14T06:54:52.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:52.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=14718], [object_number=13, tag_tid=2, entry_tid=14717], [object_number=12, tag_tid=2, entry_tid=14716], [object_number=15, tag_tid=2, entry_tid=14715]]], state=connected] 2024-09-14T06:54:52.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=14718' '!=' 'object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=14718' = 'object_number=13, tag_tid=2, entry_tid=14593' 2024-09-14T06:54:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:52.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14718 2024-09-14T06:54:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14718 2024-09-14T06:54:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:52.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:52.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:52.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:52.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:52.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:52.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:52.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:52.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:52.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:52.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:52.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:52.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:52.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:52.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:52.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:52.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:52.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:52.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=14845], [object_number=12, tag_tid=2, entry_tid=14844], [object_number=15, tag_tid=2, entry_tid=14843], [object_number=14, tag_tid=2, entry_tid=14842]]], state=connected] 2024-09-14T06:54:52.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:52.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:52.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:52.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:52.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:52.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=14845], [object_number=12, tag_tid=2, entry_tid=14844], [object_number=15, tag_tid=2, entry_tid=14843], [object_number=14, tag_tid=2, entry_tid=14842]]], state=connected] 2024-09-14T06:54:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=14845' '!=' 'object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=14845' = 'object_number=14, tag_tid=2, entry_tid=14718' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:52.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:52.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:52.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:52.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:52.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:52.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:52.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:52.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:52.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14845 2024-09-14T06:54:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14845 2024-09-14T06:54:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:52.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:52.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:52.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:52.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:52.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:52.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:52.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:53.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:53.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:53.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:53.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:53.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:53.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14971], [object_number=14, tag_tid=2, entry_tid=14970], [object_number=13, tag_tid=2, entry_tid=14969], [object_number=12, tag_tid=2, entry_tid=14968]]], state=connected] 2024-09-14T06:54:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:53.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:53.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:53.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=14971], [object_number=14, tag_tid=2, entry_tid=14970], [object_number=13, tag_tid=2, entry_tid=14969], [object_number=12, tag_tid=2, entry_tid=14968]]], state=connected] 2024-09-14T06:54:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14971' '!=' 'object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=14971' = 'object_number=13, tag_tid=2, entry_tid=14845' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:53.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14971 2024-09-14T06:54:53.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 14971 2024-09-14T06:54:53.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:53.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:53.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:53.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:53.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:53.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:53.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:53.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:53.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:53.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:53.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:53.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:53.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=15087], [object_number=14, tag_tid=2, entry_tid=15086], [object_number=13, tag_tid=2, entry_tid=15085], [object_number=12, tag_tid=2, entry_tid=15084]]], state=connected] 2024-09-14T06:54:53.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:53.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:53.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:53.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:53.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=15087], [object_number=14, tag_tid=2, entry_tid=15086], [object_number=13, tag_tid=2, entry_tid=15085], [object_number=12, tag_tid=2, entry_tid=15084]]], state=connected] 2024-09-14T06:54:53.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:53.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:53.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=15087' '!=' 'object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=15087' = 'object_number=15, tag_tid=2, entry_tid=14971' 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:53.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:53.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:53.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:53.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:53.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:53.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:53.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:53.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15087 2024-09-14T06:54:53.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15087 2024-09-14T06:54:53.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:53.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:53.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:53.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:53.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:53.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:53.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:53.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:54.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:54.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:54.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:54.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:54.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:54.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=15198], [object_number=13, tag_tid=2, entry_tid=15197], [object_number=12, tag_tid=2, entry_tid=15196], [object_number=15, tag_tid=2, entry_tid=15195]]], state=connected] 2024-09-14T06:54:54.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:54.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:54.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:54.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=15198], [object_number=13, tag_tid=2, entry_tid=15197], [object_number=12, tag_tid=2, entry_tid=15196], [object_number=15, tag_tid=2, entry_tid=15195]]], state=connected] 2024-09-14T06:54:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=15198' '!=' 'object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:54.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:54.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=15198' = 'object_number=15, tag_tid=2, entry_tid=15087' 2024-09-14T06:54:54.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:54.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:54.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15198 2024-09-14T06:54:54.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15198 2024-09-14T06:54:54.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:54.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:54.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:54.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:54.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:54.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:54.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:54.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:54.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:54.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:54.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:54.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:54.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:54.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:54.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:54.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:54.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:54.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=15316], [object_number=15, tag_tid=2, entry_tid=15315], [object_number=14, tag_tid=2, entry_tid=15314], [object_number=13, tag_tid=2, entry_tid=15313]]], state=connected] 2024-09-14T06:54:54.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:54.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:54.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:54.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:54.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:54.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:54.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:54.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:54.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:54.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:54.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=15316], [object_number=15, tag_tid=2, entry_tid=15315], [object_number=14, tag_tid=2, entry_tid=15314], [object_number=13, tag_tid=2, entry_tid=15313]]], state=connected] 2024-09-14T06:54:54.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=15316' '!=' 'object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=15316' = 'object_number=14, tag_tid=2, entry_tid=15198' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:54.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:54.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:54.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:54.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:54.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:54.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:54.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:54.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:54.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15316 2024-09-14T06:54:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15316 2024-09-14T06:54:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:55.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:55.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:55.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=15441], [object_number=12, tag_tid=2, entry_tid=15440], [object_number=15, tag_tid=2, entry_tid=15439], [object_number=14, tag_tid=2, entry_tid=15438]]], state=connected] 2024-09-14T06:54:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:55.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:55.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:55.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:55.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:55.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=15441], [object_number=12, tag_tid=2, entry_tid=15440], [object_number=15, tag_tid=2, entry_tid=15439], [object_number=14, tag_tid=2, entry_tid=15438]]], state=connected] 2024-09-14T06:54:55.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=15441' '!=' 'object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=15441' = 'object_number=12, tag_tid=2, entry_tid=15316' 2024-09-14T06:54:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:55.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:55.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:55.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:55.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:55.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:55.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:55.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:55.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15441 2024-09-14T06:54:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15441 2024-09-14T06:54:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:55.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:55.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:55.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:55.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:55.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:55.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:55.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:55.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:55.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:55.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:55.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=15568], [object_number=15, tag_tid=2, entry_tid=15567], [object_number=14, tag_tid=2, entry_tid=15566], [object_number=13, tag_tid=2, entry_tid=15565]]], state=connected] 2024-09-14T06:54:55.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:55.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:55.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:55.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:55.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=15568], [object_number=15, tag_tid=2, entry_tid=15567], [object_number=14, tag_tid=2, entry_tid=15566], [object_number=13, tag_tid=2, entry_tid=15565]]], state=connected] 2024-09-14T06:54:55.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=15568' '!=' 'object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=15568' = 'object_number=13, tag_tid=2, entry_tid=15441' 2024-09-14T06:54:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:55.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:55.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:55.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:55.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:55.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:55.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15568 2024-09-14T06:54:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15568 2024-09-14T06:54:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:55.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:55.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:55.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:55.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:55.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:55.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:55.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:55.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:55.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:55.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:55.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:56.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:56.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:56.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:56.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=15700], [object_number=15, tag_tid=2, entry_tid=15699], [object_number=14, tag_tid=2, entry_tid=15698], [object_number=13, tag_tid=2, entry_tid=15697]]], state=connected] 2024-09-14T06:54:56.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:56.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:56.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:56.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:56.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:56.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:56.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=15700], [object_number=15, tag_tid=2, entry_tid=15699], [object_number=14, tag_tid=2, entry_tid=15698], [object_number=13, tag_tid=2, entry_tid=15697]]], state=connected] 2024-09-14T06:54:56.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=15700' '!=' 'object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=15700' = 'object_number=12, tag_tid=2, entry_tid=15568' 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:56.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:56.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:56.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:56.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:56.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:56.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:56.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15700 2024-09-14T06:54:56.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15700 2024-09-14T06:54:56.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:56.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:56.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:56.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:56.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:56.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:56.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:56.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:56.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:56.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:56.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=15827], [object_number=14, tag_tid=2, entry_tid=15826], [object_number=13, tag_tid=2, entry_tid=15825], [object_number=12, tag_tid=2, entry_tid=15824]]], state=connected] 2024-09-14T06:54:56.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:56.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:56.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:56.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:56.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=15827], [object_number=14, tag_tid=2, entry_tid=15826], [object_number=13, tag_tid=2, entry_tid=15825], [object_number=12, tag_tid=2, entry_tid=15824]]], state=connected] 2024-09-14T06:54:56.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=15827' '!=' 'object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=15827' = 'object_number=12, tag_tid=2, entry_tid=15700' 2024-09-14T06:54:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:56.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:56.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:56.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:56.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:56.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:56.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15827 2024-09-14T06:54:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15827 2024-09-14T06:54:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:56.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:56.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:56.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:57.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:57.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:57.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:57.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=15947], [object_number=14, tag_tid=2, entry_tid=15946], [object_number=13, tag_tid=2, entry_tid=15945], [object_number=12, tag_tid=2, entry_tid=15944]]], state=connected] 2024-09-14T06:54:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=15947], [object_number=14, tag_tid=2, entry_tid=15946], [object_number=13, tag_tid=2, entry_tid=15945], [object_number=12, tag_tid=2, entry_tid=15944]]], state=connected] 2024-09-14T06:54:57.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=15947' '!=' 'object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=15947' = 'object_number=15, tag_tid=2, entry_tid=15827' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:57.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:57.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:57.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:57.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:57.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15947 2024-09-14T06:54:57.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 15947 2024-09-14T06:54:57.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:57.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:57.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:57.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:57.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:57.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:57.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:57.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:57.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:57.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:57.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:57.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:57.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:57.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:57.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:57.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:57.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:57.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:57.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=16070], [object_number=13, tag_tid=2, entry_tid=16069], [object_number=12, tag_tid=2, entry_tid=16068], [object_number=15, tag_tid=2, entry_tid=16067]]], state=connected] 2024-09-14T06:54:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:57.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:57.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:57.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:57.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=14, tag_tid=2, entry_tid=16070], [object_number=13, tag_tid=2, entry_tid=16069], [object_number=12, tag_tid=2, entry_tid=16068], [object_number=15, tag_tid=2, entry_tid=16067]]], state=connected] 2024-09-14T06:54:57.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=16070' '!=' 'object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=14, tag_tid=2, entry_tid=16070' = 'object_number=15, tag_tid=2, entry_tid=15947' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:57.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:57.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:57.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:57.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:57.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:57.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:57.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16070 2024-09-14T06:54:57.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16070 2024-09-14T06:54:57.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:57.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:57.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:57.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:57.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:57.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:57.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:57.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:57.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:57.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:57.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:57.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:57.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:57.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:57.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:57.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:57.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:57.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=16189], [object_number=12, tag_tid=2, entry_tid=16188], [object_number=15, tag_tid=2, entry_tid=16187], [object_number=14, tag_tid=2, entry_tid=16186]]], state=connected] 2024-09-14T06:54:58.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:58.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:58.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:58.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:58.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=16189], [object_number=12, tag_tid=2, entry_tid=16188], [object_number=15, tag_tid=2, entry_tid=16187], [object_number=14, tag_tid=2, entry_tid=16186]]], state=connected] 2024-09-14T06:54:58.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=16189' '!=' 'object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=16189' = 'object_number=14, tag_tid=2, entry_tid=16070' 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:58.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:58.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:58.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:58.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:58.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:58.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:58.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16189 2024-09-14T06:54:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16189 2024-09-14T06:54:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:58.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:58.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:58.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:58.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:58.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:58.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:58.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:58.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:58.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:58.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:58.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:58.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:58.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:58.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:58.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:58.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:58.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:58.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:58.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:58.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:58.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:58.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:58.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:58.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=16312], [object_number=15, tag_tid=2, entry_tid=16311], [object_number=14, tag_tid=2, entry_tid=16310], [object_number=13, tag_tid=2, entry_tid=16309]]], state=connected] 2024-09-14T06:54:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:58.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=2, entry_tid=16312], [object_number=15, tag_tid=2, entry_tid=16311], [object_number=14, tag_tid=2, entry_tid=16310], [object_number=13, tag_tid=2, entry_tid=16309]]], state=connected] 2024-09-14T06:54:58.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=16312' '!=' 'object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=2, entry_tid=16312' = 'object_number=13, tag_tid=2, entry_tid=16189' 2024-09-14T06:54:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16312 2024-09-14T06:54:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16312 2024-09-14T06:54:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:58.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:58.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:58.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:58.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:58.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:58.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:58.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:58.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:58.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:58.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:58.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:58.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:58.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:58.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:58.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:58.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:58.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:58.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=16429], [object_number=12, tag_tid=2, entry_tid=16428], [object_number=15, tag_tid=2, entry_tid=16427], [object_number=14, tag_tid=2, entry_tid=16426]]], state=connected] 2024-09-14T06:54:59.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:59.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=16429], [object_number=12, tag_tid=2, entry_tid=16428], [object_number=15, tag_tid=2, entry_tid=16427], [object_number=14, tag_tid=2, entry_tid=16426]]], state=connected] 2024-09-14T06:54:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=16429' '!=' 'object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=16429' = 'object_number=12, tag_tid=2, entry_tid=16312' 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:59.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:59.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:59.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:59.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:59.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16429 2024-09-14T06:54:59.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16429 2024-09-14T06:54:59.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:59.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:59.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:59.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:59.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:59.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:59.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:59.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:59.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:59.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:59.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:59.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=16553], [object_number=12, tag_tid=2, entry_tid=16552], [object_number=15, tag_tid=2, entry_tid=16551], [object_number=14, tag_tid=2, entry_tid=16550]]], state=connected] 2024-09-14T06:54:59.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:59.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:59.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:59.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:59.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=16553], [object_number=12, tag_tid=2, entry_tid=16552], [object_number=15, tag_tid=2, entry_tid=16551], [object_number=14, tag_tid=2, entry_tid=16550]]], state=connected] 2024-09-14T06:54:59.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:54:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:54:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=16553' '!=' 'object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:54:59.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=16553' = 'object_number=13, tag_tid=2, entry_tid=16429' 2024-09-14T06:54:59.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:54:59.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:59.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:54:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:54:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:54:59.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:54:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:54:59.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:54:59.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:54:59.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:54:59.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16553 2024-09-14T06:54:59.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16553 2024-09-14T06:54:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:54:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:54:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:54:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:54:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:54:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:54:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:54:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:54:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:54:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:54:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:54:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:54:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:54:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:54:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:54:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:54:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:54:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:54:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:54:59.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:54:59.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:54:59.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:54:59.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=16667], [object_number=14, tag_tid=2, entry_tid=16666], [object_number=13, tag_tid=2, entry_tid=16665], [object_number=12, tag_tid=2, entry_tid=16664]]], state=connected] 2024-09-14T06:54:59.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:54:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:54:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:54:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:54:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:54:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:54:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:54:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:54:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T06:55:00.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:00.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:00.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:00.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=16667], [object_number=14, tag_tid=2, entry_tid=16666], [object_number=13, tag_tid=2, entry_tid=16665], [object_number=12, tag_tid=2, entry_tid=16664]]], state=connected] 2024-09-14T06:55:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=16667' '!=' 'object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=16667' = 'object_number=13, tag_tid=2, entry_tid=16553' 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:00.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16667 2024-09-14T06:55:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16667 2024-09-14T06:55:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:00.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:00.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:00.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:00.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:00.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:00.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:00.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:00.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:00.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:00.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:00.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:00.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:00.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:00.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:00.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:00.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:00.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:00.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:00.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=16758], [object_number=17, tag_tid=2, entry_tid=16757], [object_number=16, tag_tid=2, entry_tid=16756], [object_number=19, tag_tid=2, entry_tid=16755]]], state=connected] 2024-09-14T06:55:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:00.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:00.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:00.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=16758], [object_number=17, tag_tid=2, entry_tid=16757], [object_number=16, tag_tid=2, entry_tid=16756], [object_number=19, tag_tid=2, entry_tid=16755]]], state=connected] 2024-09-14T06:55:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=16758' '!=' 'object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=16758' = 'object_number=15, tag_tid=2, entry_tid=16667' 2024-09-14T06:55:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:00.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:00.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:00.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:00.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:00.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:00.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:00.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:00.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16758 2024-09-14T06:55:00.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16758 2024-09-14T06:55:00.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:00.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:00.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:00.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:01.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:01.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:01.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:01.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:01.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:01.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:01.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:01.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:01.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:01.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:01.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:01.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:01.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=16927], [object_number=18, tag_tid=2, entry_tid=16926], [object_number=17, tag_tid=2, entry_tid=16925], [object_number=16, tag_tid=2, entry_tid=16924]]], state=connected] 2024-09-14T06:55:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:01.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:01.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:01.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:01.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:01.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=16927], [object_number=18, tag_tid=2, entry_tid=16926], [object_number=17, tag_tid=2, entry_tid=16925], [object_number=16, tag_tid=2, entry_tid=16924]]], state=connected] 2024-09-14T06:55:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=16927' '!=' 'object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=16927' = 'object_number=18, tag_tid=2, entry_tid=16758' 2024-09-14T06:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:01.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:01.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:01.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:01.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:01.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:01.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:01.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:01.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:01.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:01.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:01.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:01.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:01.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16927 2024-09-14T06:55:01.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 16927 2024-09-14T06:55:01.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:01.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:01.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:01.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:01.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:01.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:01.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:01.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:01.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:01.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:01.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:01.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:01.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:01.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:01.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:01.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:01.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:01.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:01.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:01.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:01.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:01.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:01.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=17099], [object_number=18, tag_tid=2, entry_tid=17098], [object_number=17, tag_tid=2, entry_tid=17097], [object_number=16, tag_tid=2, entry_tid=17096]]], state=connected] 2024-09-14T06:55:01.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:01.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:01.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:01.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:02.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:02.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:02.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:02.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=17099], [object_number=18, tag_tid=2, entry_tid=17098], [object_number=17, tag_tid=2, entry_tid=17097], [object_number=16, tag_tid=2, entry_tid=17096]]], state=connected] 2024-09-14T06:55:02.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=17099' '!=' 'object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=17099' = 'object_number=19, tag_tid=2, entry_tid=16927' 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:02.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:02.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17099 2024-09-14T06:55:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17099 2024-09-14T06:55:02.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:02.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:02.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:02.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:02.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:02.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:02.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:02.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:02.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:02.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:02.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:02.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:02.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:02.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:02.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:02.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:02.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:02.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:02.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:02.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:02.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:02.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:02.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:02.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:02.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:02.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:02.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:02.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17220], [object_number=19, tag_tid=2, entry_tid=17219], [object_number=18, tag_tid=2, entry_tid=17218], [object_number=17, tag_tid=2, entry_tid=17217]]], state=connected] 2024-09-14T06:55:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:02.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:02.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17220], [object_number=19, tag_tid=2, entry_tid=17219], [object_number=18, tag_tid=2, entry_tid=17218], [object_number=17, tag_tid=2, entry_tid=17217]]], state=connected] 2024-09-14T06:55:02.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17220' '!=' 'object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17220' = 'object_number=19, tag_tid=2, entry_tid=17099' 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:02.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:02.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:02.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17220 2024-09-14T06:55:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17220 2024-09-14T06:55:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:02.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:02.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:02.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:02.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:02.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:02.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:02.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:02.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:02.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:02.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:02.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17325], [object_number=16, tag_tid=2, entry_tid=17324], [object_number=19, tag_tid=2, entry_tid=17323], [object_number=18, tag_tid=2, entry_tid=17322]]], state=connected] 2024-09-14T06:55:02.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:02.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:02.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:02.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:02.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:02.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:02.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17325], [object_number=16, tag_tid=2, entry_tid=17324], [object_number=19, tag_tid=2, entry_tid=17323], [object_number=18, tag_tid=2, entry_tid=17322]]], state=connected] 2024-09-14T06:55:02.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17325' '!=' 'object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17325' = 'object_number=16, tag_tid=2, entry_tid=17220' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:02.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:02.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:02.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17325 2024-09-14T06:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17325 2024-09-14T06:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:02.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:03.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:03.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:03.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:03.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:03.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:03.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:03.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:03.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:03.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:03.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:03.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:03.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17433], [object_number=16, tag_tid=2, entry_tid=17432], [object_number=19, tag_tid=2, entry_tid=17431], [object_number=18, tag_tid=2, entry_tid=17430]]], state=connected] 2024-09-14T06:55:03.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17433], [object_number=16, tag_tid=2, entry_tid=17432], [object_number=19, tag_tid=2, entry_tid=17431], [object_number=18, tag_tid=2, entry_tid=17430]]], state=connected] 2024-09-14T06:55:03.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17433' '!=' 'object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17433' = 'object_number=17, tag_tid=2, entry_tid=17325' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:03.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:03.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:03.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:03.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:03.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:03.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17433 2024-09-14T06:55:03.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17433 2024-09-14T06:55:03.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:03.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:03.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:03.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:03.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:03.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:03.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:03.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:03.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:03.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:03.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:03.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:03.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:03.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:03.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:03.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:03.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:03.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:03.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:03.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17545], [object_number=16, tag_tid=2, entry_tid=17544], [object_number=19, tag_tid=2, entry_tid=17543], [object_number=18, tag_tid=2, entry_tid=17542]]], state=connected] 2024-09-14T06:55:03.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:03.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:03.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:03.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:03.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:03.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:03.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17545], [object_number=16, tag_tid=2, entry_tid=17544], [object_number=19, tag_tid=2, entry_tid=17543], [object_number=18, tag_tid=2, entry_tid=17542]]], state=connected] 2024-09-14T06:55:03.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:03.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17545' '!=' 'object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17545' = 'object_number=17, tag_tid=2, entry_tid=17433' 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:03.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:03.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:03.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:03.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17545 2024-09-14T06:55:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17545 2024-09-14T06:55:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:04.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:04.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:04.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:04.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:04.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:04.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:04.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:04.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:04.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:04.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17609], [object_number=16, tag_tid=2, entry_tid=17608], [object_number=19, tag_tid=2, entry_tid=17607], [object_number=18, tag_tid=2, entry_tid=17606]]], state=connected] 2024-09-14T06:55:04.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=17609], [object_number=16, tag_tid=2, entry_tid=17608], [object_number=19, tag_tid=2, entry_tid=17607], [object_number=18, tag_tid=2, entry_tid=17606]]], state=connected] 2024-09-14T06:55:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17609' '!=' 'object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=17609' = 'object_number=17, tag_tid=2, entry_tid=17545' 2024-09-14T06:55:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17609 2024-09-14T06:55:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17609 2024-09-14T06:55:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:04.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:04.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:04.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:04.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:04.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:04.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:04.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:04.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:04.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:04.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:04.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:04.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:04.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:04.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17728], [object_number=19, tag_tid=2, entry_tid=17727], [object_number=18, tag_tid=2, entry_tid=17726], [object_number=17, tag_tid=2, entry_tid=17725]]], state=connected] 2024-09-14T06:55:04.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:04.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:04.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:04.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:04.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:04.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17728], [object_number=19, tag_tid=2, entry_tid=17727], [object_number=18, tag_tid=2, entry_tid=17726], [object_number=17, tag_tid=2, entry_tid=17725]]], state=connected] 2024-09-14T06:55:04.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17728' '!=' 'object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17728' = 'object_number=17, tag_tid=2, entry_tid=17609' 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:04.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:04.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:04.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:04.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17728 2024-09-14T06:55:04.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17728 2024-09-14T06:55:04.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:04.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:04.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:05.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:05.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:05.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:05.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:05.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:05.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17844], [object_number=19, tag_tid=2, entry_tid=17843], [object_number=18, tag_tid=2, entry_tid=17842], [object_number=17, tag_tid=2, entry_tid=17841]]], state=connected] 2024-09-14T06:55:05.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:05.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:05.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:05.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:05.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17844], [object_number=19, tag_tid=2, entry_tid=17843], [object_number=18, tag_tid=2, entry_tid=17842], [object_number=17, tag_tid=2, entry_tid=17841]]], state=connected] 2024-09-14T06:55:05.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17844' '!=' 'object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17844' = 'object_number=16, tag_tid=2, entry_tid=17728' 2024-09-14T06:55:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:05.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:05.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:05.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:05.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:05.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17844 2024-09-14T06:55:05.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17844 2024-09-14T06:55:05.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:05.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:05.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:05.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:05.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:05.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:05.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:05.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17956], [object_number=19, tag_tid=2, entry_tid=17955], [object_number=18, tag_tid=2, entry_tid=17954], [object_number=17, tag_tid=2, entry_tid=17953]]], state=connected] 2024-09-14T06:55:05.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:05.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:05.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:05.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:05.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=17956], [object_number=19, tag_tid=2, entry_tid=17955], [object_number=18, tag_tid=2, entry_tid=17954], [object_number=17, tag_tid=2, entry_tid=17953]]], state=connected] 2024-09-14T06:55:05.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17956' '!=' 'object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:05.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=17956' = 'object_number=16, tag_tid=2, entry_tid=17844' 2024-09-14T06:55:05.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:05.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:05.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:05.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17956 2024-09-14T06:55:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 17956 2024-09-14T06:55:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:06.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:06.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:06.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:06.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:06.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:06.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:06.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:06.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:06.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:06.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:06.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:06.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:06.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=18075], [object_number=18, tag_tid=2, entry_tid=18074], [object_number=17, tag_tid=2, entry_tid=18073], [object_number=16, tag_tid=2, entry_tid=18072]]], state=connected] 2024-09-14T06:55:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:06.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:06.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:06.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=18075], [object_number=18, tag_tid=2, entry_tid=18074], [object_number=17, tag_tid=2, entry_tid=18073], [object_number=16, tag_tid=2, entry_tid=18072]]], state=connected] 2024-09-14T06:55:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=18075' '!=' 'object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:06.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:06.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=18075' = 'object_number=16, tag_tid=2, entry_tid=17956' 2024-09-14T06:55:06.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:06.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:06.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:06.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:06.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:06.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:06.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:06.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:06.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:06.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18075 2024-09-14T06:55:06.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18075 2024-09-14T06:55:06.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:06.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:06.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:06.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:06.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:06.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:06.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:06.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:06.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:06.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:06.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:06.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:06.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:06.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:06.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:06.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:06.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:06.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:06.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:06.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:06.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=18195], [object_number=18, tag_tid=2, entry_tid=18194], [object_number=17, tag_tid=2, entry_tid=18193], [object_number=16, tag_tid=2, entry_tid=18192]]], state=connected] 2024-09-14T06:55:06.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:06.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:06.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=18195], [object_number=18, tag_tid=2, entry_tid=18194], [object_number=17, tag_tid=2, entry_tid=18193], [object_number=16, tag_tid=2, entry_tid=18192]]], state=connected] 2024-09-14T06:55:06.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:06.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=18195' '!=' 'object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=18195' = 'object_number=19, tag_tid=2, entry_tid=18075' 2024-09-14T06:55:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18195 2024-09-14T06:55:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18195 2024-09-14T06:55:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:07.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:07.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:07.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:07.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:07.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:07.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:07.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:07.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:07.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:07.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:07.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=18316], [object_number=19, tag_tid=2, entry_tid=18315], [object_number=18, tag_tid=2, entry_tid=18314], [object_number=17, tag_tid=2, entry_tid=18313]]], state=connected] 2024-09-14T06:55:07.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:07.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:07.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=18316], [object_number=19, tag_tid=2, entry_tid=18315], [object_number=18, tag_tid=2, entry_tid=18314], [object_number=17, tag_tid=2, entry_tid=18313]]], state=connected] 2024-09-14T06:55:07.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=18316' '!=' 'object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=18316' = 'object_number=19, tag_tid=2, entry_tid=18195' 2024-09-14T06:55:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:07.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:07.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:07.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18316 2024-09-14T06:55:07.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18316 2024-09-14T06:55:07.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:07.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:07.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:07.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:07.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:07.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:07.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:07.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:07.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:07.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:07.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:07.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:07.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:07.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:07.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:07.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=18442], [object_number=17, tag_tid=2, entry_tid=18441], [object_number=16, tag_tid=2, entry_tid=18440], [object_number=19, tag_tid=2, entry_tid=18439]]], state=connected] 2024-09-14T06:55:07.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=18442], [object_number=17, tag_tid=2, entry_tid=18441], [object_number=16, tag_tid=2, entry_tid=18440], [object_number=19, tag_tid=2, entry_tid=18439]]], state=connected] 2024-09-14T06:55:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=18442' '!=' 'object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=18442' = 'object_number=16, tag_tid=2, entry_tid=18316' 2024-09-14T06:55:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:07.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:07.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:07.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:07.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:07.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:07.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:07.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:07.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:07.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:07.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18442 2024-09-14T06:55:07.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18442 2024-09-14T06:55:07.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:07.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:07.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:08.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:08.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:08.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:08.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:08.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:08.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:08.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:08.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:08.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:08.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:08.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:08.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:08.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=18561], [object_number=16, tag_tid=2, entry_tid=18560], [object_number=19, tag_tid=2, entry_tid=18559], [object_number=18, tag_tid=2, entry_tid=18558]]], state=connected] 2024-09-14T06:55:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=18561], [object_number=16, tag_tid=2, entry_tid=18560], [object_number=19, tag_tid=2, entry_tid=18559], [object_number=18, tag_tid=2, entry_tid=18558]]], state=connected] 2024-09-14T06:55:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:08.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:08.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=18561' '!=' 'object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:08.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:08.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=18561' = 'object_number=18, tag_tid=2, entry_tid=18442' 2024-09-14T06:55:08.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:08.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:08.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:08.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:08.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:08.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18561 2024-09-14T06:55:08.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18561 2024-09-14T06:55:08.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:08.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:08.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:08.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:08.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:08.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:08.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:08.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:08.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:08.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:08.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:08.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:08.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=18686], [object_number=17, tag_tid=2, entry_tid=18685], [object_number=16, tag_tid=2, entry_tid=18684], [object_number=19, tag_tid=2, entry_tid=18683]]], state=connected] 2024-09-14T06:55:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:08.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:08.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:08.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:08.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:08.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=18686], [object_number=17, tag_tid=2, entry_tid=18685], [object_number=16, tag_tid=2, entry_tid=18684], [object_number=19, tag_tid=2, entry_tid=18683]]], state=connected] 2024-09-14T06:55:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=18686' '!=' 'object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=18686' = 'object_number=17, tag_tid=2, entry_tid=18561' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:09.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:09.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:09.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:09.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:09.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18686 2024-09-14T06:55:09.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18686 2024-09-14T06:55:09.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:09.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:09.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:09.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:09.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:09.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:09.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:09.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:09.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:09.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:09.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:09.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:09.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:09.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:09.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:09.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=18872], [object_number=19, tag_tid=2, entry_tid=18871], [object_number=18, tag_tid=2, entry_tid=18870], [object_number=17, tag_tid=2, entry_tid=18869]]], state=connected] 2024-09-14T06:55:09.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:09.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=18872], [object_number=19, tag_tid=2, entry_tid=18871], [object_number=18, tag_tid=2, entry_tid=18870], [object_number=17, tag_tid=2, entry_tid=18869]]], state=connected] 2024-09-14T06:55:09.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=18872' '!=' 'object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=18872' = 'object_number=18, tag_tid=2, entry_tid=18686' 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:09.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:09.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18872 2024-09-14T06:55:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18872 2024-09-14T06:55:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:09.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:09.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:09.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=18999], [object_number=18, tag_tid=2, entry_tid=18998], [object_number=17, tag_tid=2, entry_tid=18997], [object_number=16, tag_tid=2, entry_tid=18996]]], state=connected] 2024-09-14T06:55:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:09.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:09.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:09.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:09.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:09.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=18999], [object_number=18, tag_tid=2, entry_tid=18998], [object_number=17, tag_tid=2, entry_tid=18997], [object_number=16, tag_tid=2, entry_tid=18996]]], state=connected] 2024-09-14T06:55:09.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:09.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:09.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:09.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=18999' '!=' 'object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:09.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=18999' = 'object_number=16, tag_tid=2, entry_tid=18872' 2024-09-14T06:55:09.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:09.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:09.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:09.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:09.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18999 2024-09-14T06:55:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 18999 2024-09-14T06:55:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:10.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:10.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:10.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:10.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:10.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:10.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:10.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:10.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:10.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:10.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:10.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:10.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:10.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:10.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:10.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:10.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:10.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:10.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:10.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:10.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:10.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:10.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:10.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:10.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:10.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:10.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19129], [object_number=16, tag_tid=2, entry_tid=19128], [object_number=19, tag_tid=2, entry_tid=19127], [object_number=18, tag_tid=2, entry_tid=19126]]], state=connected] 2024-09-14T06:55:10.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:10.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:10.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:10.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19129], [object_number=16, tag_tid=2, entry_tid=19128], [object_number=19, tag_tid=2, entry_tid=19127], [object_number=18, tag_tid=2, entry_tid=19126]]], state=connected] 2024-09-14T06:55:10.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:10.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:10.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:10.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19129' '!=' 'object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:10.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:10.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19129' = 'object_number=19, tag_tid=2, entry_tid=18999' 2024-09-14T06:55:10.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:10.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:10.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:10.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:10.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:10.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:10.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:10.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:10.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:10.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:10.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19129 2024-09-14T06:55:10.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19129 2024-09-14T06:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:10.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:10.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:10.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:10.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:10.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:10.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:10.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:10.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:10.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:10.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:10.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:10.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:10.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:11.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:11.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19305], [object_number=16, tag_tid=2, entry_tid=19304], [object_number=19, tag_tid=2, entry_tid=19303], [object_number=18, tag_tid=2, entry_tid=19302]]], state=connected] 2024-09-14T06:55:11.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19305], [object_number=16, tag_tid=2, entry_tid=19304], [object_number=19, tag_tid=2, entry_tid=19303], [object_number=18, tag_tid=2, entry_tid=19302]]], state=connected] 2024-09-14T06:55:11.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19305' '!=' 'object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19305' = 'object_number=17, tag_tid=2, entry_tid=19129' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:11.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:11.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:11.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:11.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:11.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:11.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:11.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19305 2024-09-14T06:55:11.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19305 2024-09-14T06:55:11.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:11.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:11.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:11.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:11.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:11.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:11.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:11.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:11.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:11.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:11.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19425], [object_number=16, tag_tid=2, entry_tid=19424], [object_number=19, tag_tid=2, entry_tid=19423], [object_number=18, tag_tid=2, entry_tid=19422]]], state=connected] 2024-09-14T06:55:11.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:11.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:11.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:11.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:11.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:11.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:11.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:11.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:11.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:11.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19425], [object_number=16, tag_tid=2, entry_tid=19424], [object_number=19, tag_tid=2, entry_tid=19423], [object_number=18, tag_tid=2, entry_tid=19422]]], state=connected] 2024-09-14T06:55:11.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19425' '!=' 'object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19425' = 'object_number=17, tag_tid=2, entry_tid=19305' 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:11.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:11.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:11.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19425 2024-09-14T06:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19425 2024-09-14T06:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:11.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:11.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:11.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:11.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:11.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:11.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:11.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:11.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:11.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:11.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:11.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=19544], [object_number=19, tag_tid=2, entry_tid=19543], [object_number=18, tag_tid=2, entry_tid=19542], [object_number=17, tag_tid=2, entry_tid=19541]]], state=connected] 2024-09-14T06:55:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=19544], [object_number=19, tag_tid=2, entry_tid=19543], [object_number=18, tag_tid=2, entry_tid=19542], [object_number=17, tag_tid=2, entry_tid=19541]]], state=connected] 2024-09-14T06:55:12.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=19544' '!=' 'object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=19544' = 'object_number=17, tag_tid=2, entry_tid=19425' 2024-09-14T06:55:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:12.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19544 2024-09-14T06:55:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19544 2024-09-14T06:55:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:12.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:12.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:12.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:12.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:12.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:12.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:12.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:12.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:12.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=19664], [object_number=19, tag_tid=2, entry_tid=19663], [object_number=18, tag_tid=2, entry_tid=19662], [object_number=17, tag_tid=2, entry_tid=19661]]], state=connected] 2024-09-14T06:55:12.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:12.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:12.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:12.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:12.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:12.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:12.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=16, tag_tid=2, entry_tid=19664], [object_number=19, tag_tid=2, entry_tid=19663], [object_number=18, tag_tid=2, entry_tid=19662], [object_number=17, tag_tid=2, entry_tid=19661]]], state=connected] 2024-09-14T06:55:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=19664' '!=' 'object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=16, tag_tid=2, entry_tid=19664' = 'object_number=16, tag_tid=2, entry_tid=19544' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19664 2024-09-14T06:55:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19664 2024-09-14T06:55:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:12.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:12.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:12.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:12.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:12.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:12.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:12.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:12.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:12.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=19779], [object_number=18, tag_tid=2, entry_tid=19778], [object_number=17, tag_tid=2, entry_tid=19777], [object_number=16, tag_tid=2, entry_tid=19776]]], state=connected] 2024-09-14T06:55:12.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=19779], [object_number=18, tag_tid=2, entry_tid=19778], [object_number=17, tag_tid=2, entry_tid=19777], [object_number=16, tag_tid=2, entry_tid=19776]]], state=connected] 2024-09-14T06:55:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=19779' '!=' 'object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=19779' = 'object_number=16, tag_tid=2, entry_tid=19664' 2024-09-14T06:55:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19779 2024-09-14T06:55:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19779 2024-09-14T06:55:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:13.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:13.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:13.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:13.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:13.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:13.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:13.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:13.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:13.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:13.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:13.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:13.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:13.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:13.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:13.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:13.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19889], [object_number=16, tag_tid=2, entry_tid=19888], [object_number=19, tag_tid=2, entry_tid=19887], [object_number=18, tag_tid=2, entry_tid=19886]]], state=connected] 2024-09-14T06:55:13.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:13.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:13.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:13.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:13.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:13.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:13.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:13.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:13.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:13.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=19889], [object_number=16, tag_tid=2, entry_tid=19888], [object_number=19, tag_tid=2, entry_tid=19887], [object_number=18, tag_tid=2, entry_tid=19886]]], state=connected] 2024-09-14T06:55:13.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19889' '!=' 'object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=17, tag_tid=2, entry_tid=19889' = 'object_number=19, tag_tid=2, entry_tid=19779' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:13.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:13.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:13.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:13.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:13.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:13.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:13.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:13.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19889 2024-09-14T06:55:13.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 19889 2024-09-14T06:55:13.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:13.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:13.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:13.696 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7344899922411449) > weight (0.25) 2024-09-14T06:55:13.696 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T06:55:13.696 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T06:55:13.696 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T06:55:13.697 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T06:55:13.694+0000 7fb91dc85640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 30506) UID: 1000 2024-09-14T06:55:13.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:13.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:13.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:13.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:13.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:13.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:13.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:13.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:13.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:13.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:13.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:13.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:13.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:13.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:13.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=20011], [object_number=18, tag_tid=2, entry_tid=20010], [object_number=17, tag_tid=2, entry_tid=20009], [object_number=16, tag_tid=2, entry_tid=20008]]], state=connected] 2024-09-14T06:55:13.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:13.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=19, tag_tid=2, entry_tid=20011], [object_number=18, tag_tid=2, entry_tid=20010], [object_number=17, tag_tid=2, entry_tid=20009], [object_number=16, tag_tid=2, entry_tid=20008]]], state=connected] 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=20011' '!=' 'object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=19, tag_tid=2, entry_tid=20011' = 'object_number=17, tag_tid=2, entry_tid=19889' 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=20011 2024-09-14T06:55:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 20011 2024-09-14T06:55:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:14.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:14.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:14.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:14.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:14.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:14.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:14.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:14.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:14.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:14.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:14.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:14.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:14.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:14.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=20130' '!=' 'object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=20130' = 'object_number=19, tag_tid=2, entry_tid=20011' 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:14.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T06:55:14.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:55:14.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T06:55:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:14.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:55:14.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:55:14.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:55:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=20130 2024-09-14T06:55:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 20130 2024-09-14T06:55:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:55:14.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:14.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:14.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:14.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:14.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:14.701 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T06:55:14.701 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:55:14.701 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T06:55:14.701 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T06:55:14.702 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T06:55:14.704 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T06:55:14.704 INFO:tasks.rbd_mirror_thrash:waiting for 117 secs before thrashing 2024-09-14T06:55:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:14.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:14.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:14.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=20130' '!=' 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T06:55:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:15.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:15.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:15.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:15.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:15.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:15.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:15.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:15.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:15.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:15.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:15.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:15.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:15.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:15.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:15.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:15.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:15.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:15.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:15.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:15.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:15.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:15.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:15.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:15.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:15.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:15.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:15.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:15.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=20130' '!=' 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:15.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:15.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T06:55:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:16.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:16.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:16.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:16.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:16.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:16.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:16.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:16.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:16.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:16.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:16.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:16.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:16.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:16.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:16.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:16.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:16.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:16.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:16.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:16.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:16.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:16.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:16.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:16.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:16.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:16.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:16.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:16.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:16.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:16.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:16.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:16.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:16.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=20130' '!=' 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:16.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:16.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:18.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:18.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:18.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:18.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:18.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:18.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:18.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:18.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:18.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:18.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:18.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:18.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:18.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:18.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:18.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:18.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:18.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:18.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:18.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:18.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=18, tag_tid=2, entry_tid=20130], [object_number=17, tag_tid=2, entry_tid=20129], [object_number=16, tag_tid=2, entry_tid=20128], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=18, tag_tid=2, entry_tid=20130' '!=' 'object_number=18, tag_tid=2, entry_tid=20130' 2024-09-14T06:55:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:55:18.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T06:55:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:55:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:55:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:55:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:55:20.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:55:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:55:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:55:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:55:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:55:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:55:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:55:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:55:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:55:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:55:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:55:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:55:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:55:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:55:20.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:55:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:20.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:20.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:20.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:20.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:20.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:20.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:20.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:55:20.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:55:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:55:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:55:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:20.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 4 2024-09-14T06:55:20.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 4 2024-09-14T06:55:20.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:55:20.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:20.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=17, tag_tid=3, entry_tid=1], [object_number=16, tag_tid=3, entry_tid=0], [object_number=18, tag_tid=2, entry_tid=20130], [object_number=19, tag_tid=2, entry_tid=20127]]], state=connected] 2024-09-14T06:55:20.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=17, tag_tid=3, entry_tid=1' -a 'object_number=17, tag_tid=3, entry_tid=1' = 'object_number=17, tag_tid=3, entry_tid=1' 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:20.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:20.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1 2024-09-14T06:55:20.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:55:20.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T06:55:20.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:55:21.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2024-09-14T06:55:21.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T06:55:21.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T06:55:21.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T06:55:21.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T06:55:21.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T06:55:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 06:54:43 2024 2024-09-14T06:55:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T06:55:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T06:55:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T06:55:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T06:55:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:55:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T06:55:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap1 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:55:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:55:21.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:55:21.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:55:22.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:55:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:55:23.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:55:23.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:55:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:55:23.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap1-clean 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap1-clean 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1-clean 2024-09-14T06:55:23.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T06:55:23.927 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:55:23.922+0000 7fb3e594d640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T06:55:23.929 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:55:23.926+0000 7fb3e414a640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T06:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T06:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T06:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T06:55:24.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T06:55:24.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:55:24.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T06:55:24.646 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:55:24.642+0000 7fb3e414a640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T06:55:24.646 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T06:55:24.642+0000 7fb3e594d640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 7 2024-09-14T06:55:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 19493888 bytes 2024-09-14T06:55:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 42704896 bytes 2024-09-14T06:55:27.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 64905216 bytes 2024-09-14T06:55:29.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 122272768 bytes 2024-09-14T06:55:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 165164032 bytes 2024-09-14T06:55:32.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T06:55:32.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:55:33.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T06:55:33.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:33.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:33.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:33.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:55:33.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:55:33.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:55:33.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:55:33.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:55:33.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T06:55:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T06:55:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:55:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T06:55:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":3971768.27,"entries_behind_primary":1132,"entries_per_second":246.8,"non_primary_position":{"entry_tid":18999,"object_number":19,"tag_tid":2},"primary_position":{"entry_tid":1,"object_number":17,"tag_tid":3},"seconds_until_synced":4} 2024-09-14T06:55:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T06:55:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:55:10 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:55:10' 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T06:55:33.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T06:55:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T06:55:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":3971768.27,"entries_behind_primary":1132,"entries_per_second":246.8,"non_primary_position":{"entry_tid":18999,"object_number":19,"tag_tid":2},"primary_position":{"entry_tid":1,"object_number":17,"tag_tid":3},"seconds_until_synced":4} 2024-09-14T06:55:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:55:33.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:55:33.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T06:55:33.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:55:33.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T06:55:33.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T06:55:33.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T06:55:33.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:55:10' 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 06:55:10' 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 06:55:10' 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:55:33.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T06:55:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK42144264mirror.1smithi16919.3.0-4775-g7f24e589trueOK42184267mirror.2smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":3971768.27,"entries_behind_primary":1132,"entries_per_second":246.8,"non_primary_position":{"entry_tid":18999,"object_number":19,"tag_tid":2},"primary_position":{"entry_tid":1,"object_number":17,"tag_tid":3},"seconds_until_synced":4}42104261mirror.0smithi1692024-09-14 06:55:10cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 06:55:10' 2024-09-14T06:55:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T06:55:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T06:55:33.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:55:10' 2024-09-14T06:55:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T06:55:33.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T06:55:33.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:55:33.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T06:55:33.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:55:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:55:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap2 2024-09-14T06:55:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap2 2024-09-14T06:55:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:55:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:55:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:55:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2 2024-09-14T06:55:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2024-09-14T06:55:48.701 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.25011358126392347) > weight (0.25) 2024-09-14T06:55:48.701 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T06:55:48.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T06:55:48.702 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons 2024-09-14T06:55:48.703 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T06:55:48.697+0000 7f31c2e0e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 30406) UID: 1000 2024-09-14T06:55:51.703 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T06:55:51.703 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:55:51.704 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T06:55:51.704 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T06:55:51.704 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T06:55:51.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T06:55:51.708 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing 2024-09-14T06:56:03.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T06:56:03.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:03.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:03.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:03.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:56:04.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T06:56:04.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:04.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T06:56:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T06:56:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T06:56:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:04.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T06:56:04.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:04.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:56:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:56:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:56:04.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:56:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:56:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:56:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:56:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T06:56:04.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T06:56:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T06:56:04.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T06:56:04.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T06:56:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T06:56:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T06:56:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T06:56:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T06:56:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:04.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:04.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:04.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:04.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:04.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:04.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:04.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:04.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:04.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:04.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:04.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:04.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:04.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:04.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:04.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:04.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:04.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=6905], [object_number=24, tag_tid=5, entry_tid=6904], [object_number=27, tag_tid=5, entry_tid=6903], [object_number=26, tag_tid=5, entry_tid=6902]]], state=connected] 2024-09-14T06:56:04.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:04.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:04.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:04.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:04.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:04.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:04.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=6905], [object_number=24, tag_tid=5, entry_tid=6904], [object_number=27, tag_tid=5, entry_tid=6903], [object_number=26, tag_tid=5, entry_tid=6902]]], state=connected] 2024-09-14T06:56:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=6905' '!=' '' 2024-09-14T06:56:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=6905' = '' 2024-09-14T06:56:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:05.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:05.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:05.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:05.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:05.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:05.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:05.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=6905 2024-09-14T06:56:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 6905 2024-09-14T06:56:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:05.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:05.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:05.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:05.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:05.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=7069], [object_number=24, tag_tid=5, entry_tid=7068], [object_number=27, tag_tid=5, entry_tid=7067], [object_number=26, tag_tid=5, entry_tid=7066]]], state=connected] 2024-09-14T06:56:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:05.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:05.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:05.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=7069], [object_number=24, tag_tid=5, entry_tid=7068], [object_number=27, tag_tid=5, entry_tid=7067], [object_number=26, tag_tid=5, entry_tid=7066]]], state=connected] 2024-09-14T06:56:05.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=7069' '!=' 'object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:05.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=7069' = 'object_number=25, tag_tid=5, entry_tid=6905' 2024-09-14T06:56:05.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:05.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:05.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:05.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7069 2024-09-14T06:56:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7069 2024-09-14T06:56:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:05.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:05.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:05.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:05.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:05.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:05.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:05.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:05.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:05.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:05.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:05.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:05.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:05.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:05.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:05.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7182], [object_number=25, tag_tid=5, entry_tid=7181], [object_number=24, tag_tid=5, entry_tid=7180], [object_number=27, tag_tid=5, entry_tid=7179]]], state=connected] 2024-09-14T06:56:05.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:05.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:05.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:05.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:05.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:05.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:05.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:05.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:05.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:05.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:05.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7182], [object_number=25, tag_tid=5, entry_tid=7181], [object_number=24, tag_tid=5, entry_tid=7180], [object_number=27, tag_tid=5, entry_tid=7179]]], state=connected] 2024-09-14T06:56:05.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7182' '!=' 'object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7182' = 'object_number=25, tag_tid=5, entry_tid=7069' 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:05.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:05.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:05.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:05.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:05.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:05.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:05.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:05.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7182 2024-09-14T06:56:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7182 2024-09-14T06:56:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:06.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:06.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:06.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:06.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:06.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:06.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:06.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:06.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:06.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:06.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:06.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=7293], [object_number=24, tag_tid=5, entry_tid=7292], [object_number=27, tag_tid=5, entry_tid=7291], [object_number=26, tag_tid=5, entry_tid=7290]]], state=connected] 2024-09-14T06:56:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:06.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:06.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:06.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:06.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:06.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:06.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:06.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=7293], [object_number=24, tag_tid=5, entry_tid=7292], [object_number=27, tag_tid=5, entry_tid=7291], [object_number=26, tag_tid=5, entry_tid=7290]]], state=connected] 2024-09-14T06:56:06.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=7293' '!=' 'object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=7293' = 'object_number=26, tag_tid=5, entry_tid=7182' 2024-09-14T06:56:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:06.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:06.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7293 2024-09-14T06:56:06.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7293 2024-09-14T06:56:06.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:06.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:06.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:06.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:06.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:06.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:06.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:06.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:06.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:06.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:06.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:06.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7410], [object_number=25, tag_tid=5, entry_tid=7409], [object_number=24, tag_tid=5, entry_tid=7408], [object_number=27, tag_tid=5, entry_tid=7407]]], state=connected] 2024-09-14T06:56:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:06.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:06.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7410], [object_number=25, tag_tid=5, entry_tid=7409], [object_number=24, tag_tid=5, entry_tid=7408], [object_number=27, tag_tid=5, entry_tid=7407]]], state=connected] 2024-09-14T06:56:07.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7410' '!=' 'object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7410' = 'object_number=25, tag_tid=5, entry_tid=7293' 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:07.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:07.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:07.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7410 2024-09-14T06:56:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7410 2024-09-14T06:56:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:07.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:07.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:07.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:07.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:07.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:07.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:07.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:07.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:07.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:07.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:07.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:07.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:07.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:07.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:07.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:07.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:07.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=27, tag_tid=5, entry_tid=7519], [object_number=26, tag_tid=5, entry_tid=7518], [object_number=25, tag_tid=5, entry_tid=7517], [object_number=24, tag_tid=5, entry_tid=7516]]], state=connected] 2024-09-14T06:56:07.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:07.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:07.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:07.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:07.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=27, tag_tid=5, entry_tid=7519], [object_number=26, tag_tid=5, entry_tid=7518], [object_number=25, tag_tid=5, entry_tid=7517], [object_number=24, tag_tid=5, entry_tid=7516]]], state=connected] 2024-09-14T06:56:07.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=27, tag_tid=5, entry_tid=7519' '!=' 'object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=27, tag_tid=5, entry_tid=7519' = 'object_number=26, tag_tid=5, entry_tid=7410' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:07.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:07.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:07.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:07.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7519 2024-09-14T06:56:07.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7519 2024-09-14T06:56:07.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:07.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:07.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:07.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:07.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:07.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:07.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:07.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:07.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:07.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:07.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:07.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:07.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:07.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:07.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=7613], [object_number=24, tag_tid=5, entry_tid=7612], [object_number=27, tag_tid=5, entry_tid=7611], [object_number=26, tag_tid=5, entry_tid=7610]]], state=connected] 2024-09-14T06:56:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=25, tag_tid=5, entry_tid=7613], [object_number=24, tag_tid=5, entry_tid=7612], [object_number=27, tag_tid=5, entry_tid=7611], [object_number=26, tag_tid=5, entry_tid=7610]]], state=connected] 2024-09-14T06:56:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:07.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:07.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:07.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=7613' '!=' 'object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=25, tag_tid=5, entry_tid=7613' = 'object_number=27, tag_tid=5, entry_tid=7519' 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7613 2024-09-14T06:56:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7613 2024-09-14T06:56:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:08.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:08.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:08.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:08.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:08.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:08.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:08.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:08.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:08.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:08.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:08.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:08.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:08.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:08.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:08.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:08.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:08.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:08.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:08.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:08.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:08.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:08.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:08.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:08.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:08.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7714], [object_number=25, tag_tid=5, entry_tid=7713], [object_number=24, tag_tid=5, entry_tid=7712], [object_number=27, tag_tid=5, entry_tid=7711]]], state=connected] 2024-09-14T06:56:08.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:08.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:08.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:08.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:08.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:08.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:08.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:08.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:08.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:08.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7714], [object_number=25, tag_tid=5, entry_tid=7713], [object_number=24, tag_tid=5, entry_tid=7712], [object_number=27, tag_tid=5, entry_tid=7711]]], state=connected] 2024-09-14T06:56:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7714' '!=' 'object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7714' = 'object_number=25, tag_tid=5, entry_tid=7613' 2024-09-14T06:56:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:08.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:08.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:08.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:08.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:08.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7714 2024-09-14T06:56:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7714 2024-09-14T06:56:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:08.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:08.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:08.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:08.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:08.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:08.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:08.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:08.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:08.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:08.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:08.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:08.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:08.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:08.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:08.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:08.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:08.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:08.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=7836], [object_number=27, tag_tid=5, entry_tid=7835], [object_number=26, tag_tid=5, entry_tid=7834], [object_number=25, tag_tid=5, entry_tid=7833]]], state=connected] 2024-09-14T06:56:08.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:08.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:08.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:08.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:08.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:08.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:08.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:08.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=7836], [object_number=27, tag_tid=5, entry_tid=7835], [object_number=26, tag_tid=5, entry_tid=7834], [object_number=25, tag_tid=5, entry_tid=7833]]], state=connected] 2024-09-14T06:56:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=7836' '!=' 'object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=7836' = 'object_number=26, tag_tid=5, entry_tid=7714' 2024-09-14T06:56:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7836 2024-09-14T06:56:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7836 2024-09-14T06:56:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:09.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:09.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:09.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:09.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:09.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:09.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:09.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:09.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:09.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:09.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:09.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:09.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:09.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:09.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:09.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:09.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:09.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:09.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7954], [object_number=25, tag_tid=5, entry_tid=7953], [object_number=24, tag_tid=5, entry_tid=7952], [object_number=27, tag_tid=5, entry_tid=7951]]], state=connected] 2024-09-14T06:56:09.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=7954], [object_number=25, tag_tid=5, entry_tid=7953], [object_number=24, tag_tid=5, entry_tid=7952], [object_number=27, tag_tid=5, entry_tid=7951]]], state=connected] 2024-09-14T06:56:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7954' '!=' 'object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=7954' = 'object_number=24, tag_tid=5, entry_tid=7836' 2024-09-14T06:56:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:09.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:09.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:09.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:09.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7954 2024-09-14T06:56:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 7954 2024-09-14T06:56:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:09.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:09.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:09.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:09.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:09.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:09.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:10.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:10.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:10.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:10.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:10.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:10.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:10.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8126], [object_number=25, tag_tid=5, entry_tid=8125], [object_number=24, tag_tid=5, entry_tid=8124], [object_number=27, tag_tid=5, entry_tid=8123]]], state=connected] 2024-09-14T06:56:10.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:10.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:10.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:10.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8126], [object_number=25, tag_tid=5, entry_tid=8125], [object_number=24, tag_tid=5, entry_tid=8124], [object_number=27, tag_tid=5, entry_tid=8123]]], state=connected] 2024-09-14T06:56:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8126' '!=' 'object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8126' = 'object_number=26, tag_tid=5, entry_tid=7954' 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8126 2024-09-14T06:56:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8126 2024-09-14T06:56:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:10.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:10.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:10.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:10.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:10.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:10.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:10.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:10.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:10.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:10.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:10.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:10.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8250], [object_number=25, tag_tid=5, entry_tid=8249], [object_number=24, tag_tid=5, entry_tid=8248], [object_number=27, tag_tid=5, entry_tid=8247]]], state=connected] 2024-09-14T06:56:10.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:10.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8250], [object_number=25, tag_tid=5, entry_tid=8249], [object_number=24, tag_tid=5, entry_tid=8248], [object_number=27, tag_tid=5, entry_tid=8247]]], state=connected] 2024-09-14T06:56:10.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8250' '!=' 'object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8250' = 'object_number=26, tag_tid=5, entry_tid=8126' 2024-09-14T06:56:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:10.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8250 2024-09-14T06:56:10.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8250 2024-09-14T06:56:10.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:10.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:10.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:10.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:10.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:10.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:10.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:10.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:10.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:10.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:10.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:10.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:10.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:10.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:10.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:10.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:10.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:10.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:10.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8374], [object_number=25, tag_tid=5, entry_tid=8373], [object_number=24, tag_tid=5, entry_tid=8372], [object_number=27, tag_tid=5, entry_tid=8371]]], state=connected] 2024-09-14T06:56:11.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:11.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:11.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:11.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:11.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:11.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8374], [object_number=25, tag_tid=5, entry_tid=8373], [object_number=24, tag_tid=5, entry_tid=8372], [object_number=27, tag_tid=5, entry_tid=8371]]], state=connected] 2024-09-14T06:56:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8374' '!=' 'object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:11.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:11.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8374' = 'object_number=26, tag_tid=5, entry_tid=8250' 2024-09-14T06:56:11.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:11.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:11.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:11.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:11.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:11.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8374 2024-09-14T06:56:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8374 2024-09-14T06:56:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:11.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:11.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:11.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:11.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:11.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:11.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:11.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:11.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:11.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8506], [object_number=25, tag_tid=5, entry_tid=8505], [object_number=24, tag_tid=5, entry_tid=8504], [object_number=27, tag_tid=5, entry_tid=8503]]], state=connected] 2024-09-14T06:56:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:11.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:11.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:11.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:11.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:11.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:11.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8506], [object_number=25, tag_tid=5, entry_tid=8505], [object_number=24, tag_tid=5, entry_tid=8504], [object_number=27, tag_tid=5, entry_tid=8503]]], state=connected] 2024-09-14T06:56:11.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:11.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:11.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8506' '!=' 'object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8506' = 'object_number=26, tag_tid=5, entry_tid=8374' 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:11.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:11.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:11.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:11.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8506 2024-09-14T06:56:11.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8506 2024-09-14T06:56:11.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:11.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:11.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:11.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:11.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:11.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:11.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:11.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:11.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:11.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:11.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:11.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:11.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:11.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:11.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:11.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:11.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:11.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:11.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:11.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:11.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:11.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:11.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:11.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:11.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:12.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:12.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:12.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:12.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8626], [object_number=25, tag_tid=5, entry_tid=8625], [object_number=24, tag_tid=5, entry_tid=8624], [object_number=27, tag_tid=5, entry_tid=8623]]], state=connected] 2024-09-14T06:56:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=26, tag_tid=5, entry_tid=8626], [object_number=25, tag_tid=5, entry_tid=8625], [object_number=24, tag_tid=5, entry_tid=8624], [object_number=27, tag_tid=5, entry_tid=8623]]], state=connected] 2024-09-14T06:56:12.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8626' '!=' 'object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=26, tag_tid=5, entry_tid=8626' = 'object_number=26, tag_tid=5, entry_tid=8506' 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8626 2024-09-14T06:56:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8626 2024-09-14T06:56:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:12.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:12.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:12.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:12.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:12.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:12.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:12.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:12.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=8736], [object_number=27, tag_tid=5, entry_tid=8735], [object_number=26, tag_tid=5, entry_tid=8734], [object_number=25, tag_tid=5, entry_tid=8733]]], state=connected] 2024-09-14T06:56:12.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:12.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:12.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:12.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:12.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:12.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:12.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:12.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:12.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:12.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=8736], [object_number=27, tag_tid=5, entry_tid=8735], [object_number=26, tag_tid=5, entry_tid=8734], [object_number=25, tag_tid=5, entry_tid=8733]]], state=connected] 2024-09-14T06:56:12.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=8736' '!=' 'object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=8736' = 'object_number=26, tag_tid=5, entry_tid=8626' 2024-09-14T06:56:12.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:12.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:12.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:12.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:12.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:12.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:12.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:12.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:12.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:12.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:12.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:12.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8736 2024-09-14T06:56:12.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8736 2024-09-14T06:56:12.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:12.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:12.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:12.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:12.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:12.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:12.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:12.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:12.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:12.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=8856], [object_number=27, tag_tid=5, entry_tid=8855], [object_number=26, tag_tid=5, entry_tid=8854], [object_number=25, tag_tid=5, entry_tid=8853]]], state=connected] 2024-09-14T06:56:12.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:12.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:12.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:12.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:12.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:12.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:12.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=8856], [object_number=27, tag_tid=5, entry_tid=8855], [object_number=26, tag_tid=5, entry_tid=8854], [object_number=25, tag_tid=5, entry_tid=8853]]], state=connected] 2024-09-14T06:56:13.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=8856' '!=' 'object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=8856' = 'object_number=24, tag_tid=5, entry_tid=8736' 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:13.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:13.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:13.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:13.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8856 2024-09-14T06:56:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8856 2024-09-14T06:56:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:13.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:13.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:13.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:13.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:13.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:13.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:13.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:13.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:13.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:13.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=8988], [object_number=27, tag_tid=5, entry_tid=8987], [object_number=26, tag_tid=5, entry_tid=8986], [object_number=25, tag_tid=5, entry_tid=8985]]], state=connected] 2024-09-14T06:56:13.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:13.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:13.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:13.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:13.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:13.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:13.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:13.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=8988], [object_number=27, tag_tid=5, entry_tid=8987], [object_number=26, tag_tid=5, entry_tid=8986], [object_number=25, tag_tid=5, entry_tid=8985]]], state=connected] 2024-09-14T06:56:13.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=8988' '!=' 'object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=8988' = 'object_number=24, tag_tid=5, entry_tid=8856' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:13.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:13.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:13.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:13.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:13.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:13.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:13.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8988 2024-09-14T06:56:13.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 8988 2024-09-14T06:56:13.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:13.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:13.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:13.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:13.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:13.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:13.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:13.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:13.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:13.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:13.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:13.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:13.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:13.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:13.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:13.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:13.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:13.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:13.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:13.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:13.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:13.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:13.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=9100], [object_number=27, tag_tid=5, entry_tid=9099], [object_number=26, tag_tid=5, entry_tid=9098], [object_number=25, tag_tid=5, entry_tid=9097]]], state=connected] 2024-09-14T06:56:13.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 6 2024-09-14T06:56:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=24, tag_tid=5, entry_tid=9100], [object_number=27, tag_tid=5, entry_tid=9099], [object_number=26, tag_tid=5, entry_tid=9098], [object_number=25, tag_tid=5, entry_tid=9097]]], state=connected] 2024-09-14T06:56:14.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=9100' '!=' 'object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=24, tag_tid=5, entry_tid=9100' = 'object_number=24, tag_tid=5, entry_tid=8988' 2024-09-14T06:56:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9100 2024-09-14T06:56:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9100 2024-09-14T06:56:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:14.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:14.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:14.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:14.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:14.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:14.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:14.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:14.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:14.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:14.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:14.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:14.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:14.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:14.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:14.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:14.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=9174], [object_number=29, tag_tid=5, entry_tid=9173], [object_number=28, tag_tid=5, entry_tid=9172], [object_number=31, tag_tid=5, entry_tid=9171]]], state=connected] 2024-09-14T06:56:14.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:14.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:14.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:14.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:14.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:14.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:14.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=9174], [object_number=29, tag_tid=5, entry_tid=9173], [object_number=28, tag_tid=5, entry_tid=9172], [object_number=31, tag_tid=5, entry_tid=9171]]], state=connected] 2024-09-14T06:56:14.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=9174' '!=' 'object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=9174' = 'object_number=24, tag_tid=5, entry_tid=9100' 2024-09-14T06:56:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9174 2024-09-14T06:56:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9174 2024-09-14T06:56:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:14.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:14.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:14.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:14.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:14.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:14.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:14.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:14.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:14.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:14.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:14.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:14.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:14.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:14.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:14.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:15.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=9324], [object_number=31, tag_tid=5, entry_tid=9323], [object_number=30, tag_tid=5, entry_tid=9322], [object_number=29, tag_tid=5, entry_tid=9321]]], state=connected] 2024-09-14T06:56:15.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:15.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=9324], [object_number=31, tag_tid=5, entry_tid=9323], [object_number=30, tag_tid=5, entry_tid=9322], [object_number=29, tag_tid=5, entry_tid=9321]]], state=connected] 2024-09-14T06:56:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=9324' '!=' 'object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=9324' = 'object_number=30, tag_tid=5, entry_tid=9174' 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:15.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9324 2024-09-14T06:56:15.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9324 2024-09-14T06:56:15.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:15.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:15.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:15.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:15.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:15.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:15.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:15.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:15.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:15.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:15.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:15.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:15.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:15.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:15.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:15.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:15.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9433], [object_number=28, tag_tid=5, entry_tid=9432], [object_number=31, tag_tid=5, entry_tid=9431], [object_number=30, tag_tid=5, entry_tid=9430]]], state=connected] 2024-09-14T06:56:15.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:15.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9433], [object_number=28, tag_tid=5, entry_tid=9432], [object_number=31, tag_tid=5, entry_tid=9431], [object_number=30, tag_tid=5, entry_tid=9430]]], state=connected] 2024-09-14T06:56:15.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:15.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:15.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:15.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9433' '!=' 'object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:15.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9433' = 'object_number=28, tag_tid=5, entry_tid=9324' 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:15.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9433 2024-09-14T06:56:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9433 2024-09-14T06:56:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:15.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:15.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:15.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:15.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:15.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:15.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:15.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:16.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9541], [object_number=28, tag_tid=5, entry_tid=9540], [object_number=31, tag_tid=5, entry_tid=9539], [object_number=30, tag_tid=5, entry_tid=9538]]], state=connected] 2024-09-14T06:56:16.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:16.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:16.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:16.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:16.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:16.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:16.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9541], [object_number=28, tag_tid=5, entry_tid=9540], [object_number=31, tag_tid=5, entry_tid=9539], [object_number=30, tag_tid=5, entry_tid=9538]]], state=connected] 2024-09-14T06:56:16.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9541' '!=' 'object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9541' = 'object_number=29, tag_tid=5, entry_tid=9433' 2024-09-14T06:56:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:16.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:16.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:16.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:16.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:16.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:16.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:16.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9541 2024-09-14T06:56:16.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9541 2024-09-14T06:56:16.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:16.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:16.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:16.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:16.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:16.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:16.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:16.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:16.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:16.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:16.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:16.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:16.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:16.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9653], [object_number=28, tag_tid=5, entry_tid=9652], [object_number=31, tag_tid=5, entry_tid=9651], [object_number=30, tag_tid=5, entry_tid=9650]]], state=connected] 2024-09-14T06:56:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:16.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:16.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:16.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:16.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9653], [object_number=28, tag_tid=5, entry_tid=9652], [object_number=31, tag_tid=5, entry_tid=9651], [object_number=30, tag_tid=5, entry_tid=9650]]], state=connected] 2024-09-14T06:56:16.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9653' '!=' 'object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9653' = 'object_number=29, tag_tid=5, entry_tid=9541' 2024-09-14T06:56:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:16.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:16.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:16.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:16.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:16.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:16.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:16.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9653 2024-09-14T06:56:16.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9653 2024-09-14T06:56:16.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:16.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:16.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:16.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:16.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:16.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:16.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:17.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:17.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:17.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:17.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:17.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:17.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:17.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:17.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:17.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9765], [object_number=28, tag_tid=5, entry_tid=9764], [object_number=31, tag_tid=5, entry_tid=9763], [object_number=30, tag_tid=5, entry_tid=9762]]], state=connected] 2024-09-14T06:56:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:17.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:17.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:17.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:17.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=9765], [object_number=28, tag_tid=5, entry_tid=9764], [object_number=31, tag_tid=5, entry_tid=9763], [object_number=30, tag_tid=5, entry_tid=9762]]], state=connected] 2024-09-14T06:56:17.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9765' '!=' 'object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=9765' = 'object_number=29, tag_tid=5, entry_tid=9653' 2024-09-14T06:56:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:17.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:17.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:17.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:17.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9765 2024-09-14T06:56:17.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9765 2024-09-14T06:56:17.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:17.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:17.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:17.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:17.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:17.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:17.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:17.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:17.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:17.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:17.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:17.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:17.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=9876], [object_number=31, tag_tid=5, entry_tid=9875], [object_number=30, tag_tid=5, entry_tid=9874], [object_number=29, tag_tid=5, entry_tid=9873]]], state=connected] 2024-09-14T06:56:17.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:17.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:17.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:17.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:17.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=9876], [object_number=31, tag_tid=5, entry_tid=9875], [object_number=30, tag_tid=5, entry_tid=9874], [object_number=29, tag_tid=5, entry_tid=9873]]], state=connected] 2024-09-14T06:56:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=9876' '!=' 'object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=9876' = 'object_number=29, tag_tid=5, entry_tid=9765' 2024-09-14T06:56:17.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:17.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:17.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:17.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:17.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:17.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:17.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:17.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9876 2024-09-14T06:56:17.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9876 2024-09-14T06:56:17.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:17.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:17.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:17.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:17.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:17.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:17.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:17.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:17.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:17.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:17.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:17.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:17.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:17.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:18.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:18.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:18.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=9990], [object_number=29, tag_tid=5, entry_tid=9989], [object_number=28, tag_tid=5, entry_tid=9988], [object_number=31, tag_tid=5, entry_tid=9987]]], state=connected] 2024-09-14T06:56:18.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:18.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:18.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:18.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:18.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:18.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=9990], [object_number=29, tag_tid=5, entry_tid=9989], [object_number=28, tag_tid=5, entry_tid=9988], [object_number=31, tag_tid=5, entry_tid=9987]]], state=connected] 2024-09-14T06:56:18.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=9990' '!=' 'object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=9990' = 'object_number=28, tag_tid=5, entry_tid=9876' 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:18.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:18.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:18.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:18.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:18.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:18.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:18.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:18.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9990 2024-09-14T06:56:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 9990 2024-09-14T06:56:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:18.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:18.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:18.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:18.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:18.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:18.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:18.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:18.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=10097], [object_number=28, tag_tid=5, entry_tid=10096], [object_number=31, tag_tid=5, entry_tid=10095], [object_number=30, tag_tid=5, entry_tid=10094]]], state=connected] 2024-09-14T06:56:18.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:18.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:18.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:18.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:18.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:18.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:18.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:18.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:18.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=10097], [object_number=28, tag_tid=5, entry_tid=10096], [object_number=31, tag_tid=5, entry_tid=10095], [object_number=30, tag_tid=5, entry_tid=10094]]], state=connected] 2024-09-14T06:56:18.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=10097' '!=' 'object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=10097' = 'object_number=30, tag_tid=5, entry_tid=9990' 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:18.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:18.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:18.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:18.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:18.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:18.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10097 2024-09-14T06:56:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10097 2024-09-14T06:56:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:18.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:18.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:18.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:18.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:18.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:18.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:18.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:18.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:19.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:19.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:19.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:19.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10202], [object_number=29, tag_tid=5, entry_tid=10201], [object_number=28, tag_tid=5, entry_tid=10200], [object_number=31, tag_tid=5, entry_tid=10199]]], state=connected] 2024-09-14T06:56:19.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:19.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:19.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10202], [object_number=29, tag_tid=5, entry_tid=10201], [object_number=28, tag_tid=5, entry_tid=10200], [object_number=31, tag_tid=5, entry_tid=10199]]], state=connected] 2024-09-14T06:56:19.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10202' '!=' 'object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:19.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10202' = 'object_number=29, tag_tid=5, entry_tid=10097' 2024-09-14T06:56:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10202 2024-09-14T06:56:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10202 2024-09-14T06:56:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:19.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:19.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:19.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:19.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:19.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:19.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:19.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:19.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:19.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:19.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:19.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=10328], [object_number=31, tag_tid=5, entry_tid=10327], [object_number=30, tag_tid=5, entry_tid=10326], [object_number=29, tag_tid=5, entry_tid=10325]]], state=connected] 2024-09-14T06:56:19.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:19.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:19.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:19.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:19.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:19.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:19.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:19.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:19.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:19.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=10328], [object_number=31, tag_tid=5, entry_tid=10327], [object_number=30, tag_tid=5, entry_tid=10326], [object_number=29, tag_tid=5, entry_tid=10325]]], state=connected] 2024-09-14T06:56:19.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=10328' '!=' 'object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=10328' = 'object_number=30, tag_tid=5, entry_tid=10202' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:19.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:19.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:19.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:19.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:19.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:19.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:19.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10328 2024-09-14T06:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10328 2024-09-14T06:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:19.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:19.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:19.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:19.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:19.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:19.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:19.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:19.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:19.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:19.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:20.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:20.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:20.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:20.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=31, tag_tid=5, entry_tid=10447], [object_number=30, tag_tid=5, entry_tid=10446], [object_number=29, tag_tid=5, entry_tid=10445], [object_number=28, tag_tid=5, entry_tid=10444]]], state=connected] 2024-09-14T06:56:20.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:20.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:20.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:20.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:20.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:20.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:20.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=31, tag_tid=5, entry_tid=10447], [object_number=30, tag_tid=5, entry_tid=10446], [object_number=29, tag_tid=5, entry_tid=10445], [object_number=28, tag_tid=5, entry_tid=10444]]], state=connected] 2024-09-14T06:56:20.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=31, tag_tid=5, entry_tid=10447' '!=' 'object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:20.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:20.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=31, tag_tid=5, entry_tid=10447' = 'object_number=28, tag_tid=5, entry_tid=10328' 2024-09-14T06:56:20.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:20.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:20.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:20.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:20.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:20.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:20.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:20.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:20.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10447 2024-09-14T06:56:20.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10447 2024-09-14T06:56:20.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:20.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:20.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:20.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:20.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:20.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:20.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:20.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10570], [object_number=29, tag_tid=5, entry_tid=10569], [object_number=28, tag_tid=5, entry_tid=10568], [object_number=31, tag_tid=5, entry_tid=10567]]], state=connected] 2024-09-14T06:56:20.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:20.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:20.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:20.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:20.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:20.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:20.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:20.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10570], [object_number=29, tag_tid=5, entry_tid=10569], [object_number=28, tag_tid=5, entry_tid=10568], [object_number=31, tag_tid=5, entry_tid=10567]]], state=connected] 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10570' '!=' 'object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10570' = 'object_number=31, tag_tid=5, entry_tid=10447' 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:20.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:20.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10570 2024-09-14T06:56:20.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10570 2024-09-14T06:56:20.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:20.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:20.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:20.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:20.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:20.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:20.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:20.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:20.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:20.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:20.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:20.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:20.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:20.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:20.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:20.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:20.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:20.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:20.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=10680], [object_number=31, tag_tid=5, entry_tid=10679], [object_number=30, tag_tid=5, entry_tid=10678], [object_number=29, tag_tid=5, entry_tid=10677]]], state=connected] 2024-09-14T06:56:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=10680], [object_number=31, tag_tid=5, entry_tid=10679], [object_number=30, tag_tid=5, entry_tid=10678], [object_number=29, tag_tid=5, entry_tid=10677]]], state=connected] 2024-09-14T06:56:21.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=10680' '!=' 'object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=10680' = 'object_number=30, tag_tid=5, entry_tid=10570' 2024-09-14T06:56:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:21.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:21.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:21.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:21.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10680 2024-09-14T06:56:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10680 2024-09-14T06:56:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:21.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:21.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:21.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:21.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:21.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:21.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:21.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:21.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:21.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:21.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:21.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:21.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:21.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:21.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:21.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:21.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:21.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:21.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:21.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:21.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:21.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:21.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:21.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:21.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10790], [object_number=29, tag_tid=5, entry_tid=10789], [object_number=28, tag_tid=5, entry_tid=10788], [object_number=31, tag_tid=5, entry_tid=10787]]], state=connected] 2024-09-14T06:56:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10790], [object_number=29, tag_tid=5, entry_tid=10789], [object_number=28, tag_tid=5, entry_tid=10788], [object_number=31, tag_tid=5, entry_tid=10787]]], state=connected] 2024-09-14T06:56:21.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10790' '!=' 'object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10790' = 'object_number=28, tag_tid=5, entry_tid=10680' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:21.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:21.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:21.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10790 2024-09-14T06:56:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10790 2024-09-14T06:56:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:21.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:21.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:21.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:21.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:21.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:21.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:21.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:21.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:21.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:21.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:21.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:21.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:21.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:21.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:21.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:21.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:21.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:21.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:21.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:21.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:21.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10902], [object_number=29, tag_tid=5, entry_tid=10901], [object_number=28, tag_tid=5, entry_tid=10900], [object_number=31, tag_tid=5, entry_tid=10899]]], state=connected] 2024-09-14T06:56:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:21.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:21.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:21.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=10902], [object_number=29, tag_tid=5, entry_tid=10901], [object_number=28, tag_tid=5, entry_tid=10900], [object_number=31, tag_tid=5, entry_tid=10899]]], state=connected] 2024-09-14T06:56:22.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10902' '!=' 'object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=10902' = 'object_number=30, tag_tid=5, entry_tid=10790' 2024-09-14T06:56:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:22.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:22.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10902 2024-09-14T06:56:22.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 10902 2024-09-14T06:56:22.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:22.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:22.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:22.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:22.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:22.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:22.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:22.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:22.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:22.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:22.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:22.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:22.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:22.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:22.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:22.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:22.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:22.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:22.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:22.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:22.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11024], [object_number=31, tag_tid=5, entry_tid=11023], [object_number=30, tag_tid=5, entry_tid=11022], [object_number=29, tag_tid=5, entry_tid=11021]]], state=connected] 2024-09-14T06:56:22.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:22.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:22.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:22.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:22.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11024], [object_number=31, tag_tid=5, entry_tid=11023], [object_number=30, tag_tid=5, entry_tid=11022], [object_number=29, tag_tid=5, entry_tid=11021]]], state=connected] 2024-09-14T06:56:22.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11024' '!=' 'object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:22.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11024' = 'object_number=30, tag_tid=5, entry_tid=10902' 2024-09-14T06:56:22.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:22.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:22.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11024 2024-09-14T06:56:22.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11024 2024-09-14T06:56:22.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:22.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:22.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:22.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:22.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:22.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:22.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:22.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:22.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:22.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:22.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:22.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:22.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:22.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=11161], [object_number=28, tag_tid=5, entry_tid=11160], [object_number=31, tag_tid=5, entry_tid=11159], [object_number=30, tag_tid=5, entry_tid=11158]]], state=connected] 2024-09-14T06:56:22.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:22.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:22.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:22.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:22.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:22.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:22.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=11161], [object_number=28, tag_tid=5, entry_tid=11160], [object_number=31, tag_tid=5, entry_tid=11159], [object_number=30, tag_tid=5, entry_tid=11158]]], state=connected] 2024-09-14T06:56:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:22.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:22.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:22.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=11161' '!=' 'object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:22.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=11161' = 'object_number=28, tag_tid=5, entry_tid=11024' 2024-09-14T06:56:22.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:22.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:22.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:22.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:22.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:22.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:22.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:22.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:22.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:22.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11161 2024-09-14T06:56:22.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11161 2024-09-14T06:56:22.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:22.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:22.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:23.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:23.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11296], [object_number=31, tag_tid=5, entry_tid=11295], [object_number=30, tag_tid=5, entry_tid=11294], [object_number=29, tag_tid=5, entry_tid=11293]]], state=connected] 2024-09-14T06:56:23.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:23.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11296], [object_number=31, tag_tid=5, entry_tid=11295], [object_number=30, tag_tid=5, entry_tid=11294], [object_number=29, tag_tid=5, entry_tid=11293]]], state=connected] 2024-09-14T06:56:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11296' '!=' 'object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11296' = 'object_number=29, tag_tid=5, entry_tid=11161' 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11296 2024-09-14T06:56:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11296 2024-09-14T06:56:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:23.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:23.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:23.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:23.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:23.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:23.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:23.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:23.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:23.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11408], [object_number=31, tag_tid=5, entry_tid=11407], [object_number=30, tag_tid=5, entry_tid=11406], [object_number=29, tag_tid=5, entry_tid=11405]]], state=connected] 2024-09-14T06:56:23.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:23.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:23.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:23.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:23.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:23.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11408], [object_number=31, tag_tid=5, entry_tid=11407], [object_number=30, tag_tid=5, entry_tid=11406], [object_number=29, tag_tid=5, entry_tid=11405]]], state=connected] 2024-09-14T06:56:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11408' '!=' 'object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11408' = 'object_number=28, tag_tid=5, entry_tid=11296' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:23.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:23.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:23.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:23.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:23.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11408 2024-09-14T06:56:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11408 2024-09-14T06:56:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:24.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:24.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:24.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:24.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:24.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:24.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:24.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:24.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:24.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:24.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:24.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:24.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=11517], [object_number=28, tag_tid=5, entry_tid=11516], [object_number=31, tag_tid=5, entry_tid=11515], [object_number=30, tag_tid=5, entry_tid=11514]]], state=connected] 2024-09-14T06:56:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:24.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=11517], [object_number=28, tag_tid=5, entry_tid=11516], [object_number=31, tag_tid=5, entry_tid=11515], [object_number=30, tag_tid=5, entry_tid=11514]]], state=connected] 2024-09-14T06:56:24.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=11517' '!=' 'object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=11517' = 'object_number=28, tag_tid=5, entry_tid=11408' 2024-09-14T06:56:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11517 2024-09-14T06:56:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11517 2024-09-14T06:56:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:24.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:24.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:24.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:24.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:24.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:24.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:24.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=11622], [object_number=29, tag_tid=5, entry_tid=11621], [object_number=28, tag_tid=5, entry_tid=11620], [object_number=31, tag_tid=5, entry_tid=11619]]], state=connected] 2024-09-14T06:56:24.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=11622], [object_number=29, tag_tid=5, entry_tid=11621], [object_number=28, tag_tid=5, entry_tid=11620], [object_number=31, tag_tid=5, entry_tid=11619]]], state=connected] 2024-09-14T06:56:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=11622' '!=' 'object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=11622' = 'object_number=29, tag_tid=5, entry_tid=11517' 2024-09-14T06:56:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11622 2024-09-14T06:56:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11622 2024-09-14T06:56:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:25.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:25.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:25.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:25.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:25.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:25.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:25.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:25.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:25.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:25.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:25.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:25.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:25.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:25.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:25.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=11746], [object_number=29, tag_tid=5, entry_tid=11745], [object_number=28, tag_tid=5, entry_tid=11744], [object_number=31, tag_tid=5, entry_tid=11743]]], state=connected] 2024-09-14T06:56:25.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:25.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:25.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:25.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:25.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:25.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:25.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:25.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:25.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=11746], [object_number=29, tag_tid=5, entry_tid=11745], [object_number=28, tag_tid=5, entry_tid=11744], [object_number=31, tag_tid=5, entry_tid=11743]]], state=connected] 2024-09-14T06:56:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=11746' '!=' 'object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=11746' = 'object_number=30, tag_tid=5, entry_tid=11622' 2024-09-14T06:56:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:25.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11746 2024-09-14T06:56:25.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11746 2024-09-14T06:56:25.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:25.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:25.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:25.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:25.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:25.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:25.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:25.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:25.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:25.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=11870], [object_number=29, tag_tid=5, entry_tid=11869], [object_number=28, tag_tid=5, entry_tid=11868], [object_number=31, tag_tid=5, entry_tid=11867]]], state=connected] 2024-09-14T06:56:25.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:25.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:25.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:25.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:25.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:25.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:25.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:25.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:25.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=30, tag_tid=5, entry_tid=11870], [object_number=29, tag_tid=5, entry_tid=11869], [object_number=28, tag_tid=5, entry_tid=11868], [object_number=31, tag_tid=5, entry_tid=11867]]], state=connected] 2024-09-14T06:56:25.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=11870' '!=' 'object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=30, tag_tid=5, entry_tid=11870' = 'object_number=30, tag_tid=5, entry_tid=11746' 2024-09-14T06:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11870 2024-09-14T06:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11870 2024-09-14T06:56:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:26.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:26.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:26.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:26.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:26.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:26.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:26.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:26.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:26.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:26.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:26.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:26.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:26.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:26.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:26.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11988], [object_number=31, tag_tid=5, entry_tid=11987], [object_number=30, tag_tid=5, entry_tid=11986], [object_number=29, tag_tid=5, entry_tid=11985]]], state=connected] 2024-09-14T06:56:26.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:26.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:26.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:26.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:26.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:26.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:26.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:26.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=11988], [object_number=31, tag_tid=5, entry_tid=11987], [object_number=30, tag_tid=5, entry_tid=11986], [object_number=29, tag_tid=5, entry_tid=11985]]], state=connected] 2024-09-14T06:56:26.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11988' '!=' 'object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=11988' = 'object_number=30, tag_tid=5, entry_tid=11870' 2024-09-14T06:56:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:26.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:26.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:26.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:26.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:26.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11988 2024-09-14T06:56:26.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 11988 2024-09-14T06:56:26.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:26.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:26.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:26.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:26.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:26.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:26.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12101], [object_number=28, tag_tid=5, entry_tid=12100], [object_number=31, tag_tid=5, entry_tid=12099], [object_number=30, tag_tid=5, entry_tid=12098]]], state=connected] 2024-09-14T06:56:26.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:26.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:26.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:26.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:26.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:26.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:26.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12101], [object_number=28, tag_tid=5, entry_tid=12100], [object_number=31, tag_tid=5, entry_tid=12099], [object_number=30, tag_tid=5, entry_tid=12098]]], state=connected] 2024-09-14T06:56:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:26.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12101' '!=' 'object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12101' = 'object_number=28, tag_tid=5, entry_tid=11988' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:26.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:26.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:26.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12101 2024-09-14T06:56:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 12101 2024-09-14T06:56:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:26.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:26.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:27.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:27.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:27.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=12216], [object_number=31, tag_tid=5, entry_tid=12215], [object_number=30, tag_tid=5, entry_tid=12214], [object_number=29, tag_tid=5, entry_tid=12213]]], state=connected] 2024-09-14T06:56:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:27.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:27.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=28, tag_tid=5, entry_tid=12216], [object_number=31, tag_tid=5, entry_tid=12215], [object_number=30, tag_tid=5, entry_tid=12214], [object_number=29, tag_tid=5, entry_tid=12213]]], state=connected] 2024-09-14T06:56:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=12216' '!=' 'object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=28, tag_tid=5, entry_tid=12216' = 'object_number=29, tag_tid=5, entry_tid=12101' 2024-09-14T06:56:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:27.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:27.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:27.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12216 2024-09-14T06:56:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 12216 2024-09-14T06:56:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:27.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:27.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:27.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:27.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:27.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:27.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:27.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:27.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:27.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:27.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:27.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:27.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:27.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:27.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:27.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:27.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:27.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:27.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:27.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:27.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:27.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:27.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:27.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:27.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:27.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12229' '!=' 'object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12229' = 'object_number=28, tag_tid=5, entry_tid=12216' 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=6 2024-09-14T06:56:27.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:27.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:56:27.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=5 2024-09-14T06:56:27.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:27.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:27.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:56:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:56:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:56:27.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12229 2024-09-14T06:56:27.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 6 = 5 -a 1 -le 12229 2024-09-14T06:56:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:56:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:28.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:28.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:28.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:28.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:28.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:28.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:28.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:28.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:28.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:28.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:28.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:28.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:28.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:28.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:28.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:28.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12229' '!=' 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T06:56:28.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:28.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:28.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:28.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:28.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:28.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:28.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:28.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:28.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:28.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:28.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:28.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12229' '!=' 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:28.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:28.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T06:56:29.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:29.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:29.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:29.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:29.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:29.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:29.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:29.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:29.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:29.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:29.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:29.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:29.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:29.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:29.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:29.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:30.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:30.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:30.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:30.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:30.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:30.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:30.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:30.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:30.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:30.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:30.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:30.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:30.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:30.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:30.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12229' '!=' 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:30.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:30.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T06:56:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:31.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:31.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:31.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:31.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:31.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:31.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:31.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:31.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:32.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:32.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:32.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:32.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:32.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:32.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:32.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:32.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:32.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:32.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:32.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:32.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:32.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=5, entry_tid=12229], [object_number=28, tag_tid=5, entry_tid=12228], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=29, tag_tid=5, entry_tid=12229' '!=' 'object_number=29, tag_tid=5, entry_tid=12229' 2024-09-14T06:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:56:32.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:56:34.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:56:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:56:34.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:56:34.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:56:34.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:56:34.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:56:34.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:56:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:56:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:56:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:56:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:56:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:56:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:56:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:56:34.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:34.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:34.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:56:34.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:56:34.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:56:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:34.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:34.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:34.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:34.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:34.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:34.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:34.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:34.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:34.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:34.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:34.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:34.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:56:34.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:56:34.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:34.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:56:34.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 7 2024-09-14T06:56:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 7 2024-09-14T06:56:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:56:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:34.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=29, tag_tid=6, entry_tid=1], [object_number=28, tag_tid=6, entry_tid=0], [object_number=31, tag_tid=5, entry_tid=12227], [object_number=30, tag_tid=5, entry_tid=12226]]], state=connected] 2024-09-14T06:56:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=29, tag_tid=6, entry_tid=1' -a 'object_number=29, tag_tid=6, entry_tid=1' = 'object_number=29, tag_tid=6, entry_tid=1' 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2 2024-09-14T06:56:34.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:56:34.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T06:56:34.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:56:35.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2024-09-14T06:56:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T06:56:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 3 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 06:55:22 2024 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 06:55:44 2024 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T06:56:35.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T06:56:35.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T06:56:35.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T06:56:35.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:56:35.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T06:56:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap1-clean ']' 2024-09-14T06:56:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap1-clean 2024-09-14T06:56:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1-clean 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:56:35.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:56:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:56:37.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap2 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:37.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:56:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:56:38.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:38.889 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T06:56:38.884+0000 7f0de5b8b640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:56:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:56:39.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:39.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap2-clean 2024-09-14T06:56:39.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap2-clean 2024-09-14T06:56:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:56:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2-clean 2024-09-14T06:56:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T06:56:40.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T06:56:40.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T06:56:40.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:56:40.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:40.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:40.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T06:56:40.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T06:56:40.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:56:40.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T06:56:41.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 42745856 bytes 2024-09-14T06:56:44.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 102136832 bytes 2024-09-14T06:56:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 118874112 bytes 2024-09-14T06:56:46.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 138477568 bytes 2024-09-14T06:56:49.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 233074688 bytes 2024-09-14T06:56:50.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:56:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:56:51.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:56:51.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:56:51.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":2915970.23,"entries_behind_primary":0,"entries_per_second":182.33,"non_primary_position":{"entry_tid":1,"object_number":29,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":29,"tag_tid":6}} 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:56:40 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T06:56:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:56:40' 2024-09-14T06:56:51.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T06:56:51.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T06:56:51.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T06:56:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":2915970.23,"entries_behind_primary":0,"entries_per_second":182.33,"non_primary_position":{"entry_tid":1,"object_number":29,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":29,"tag_tid":6}} 2024-09-14T06:56:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:56:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:56:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T06:56:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:56:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T06:56:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T06:56:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T06:56:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T06:56:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:56:40' 2024-09-14T06:56:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 06:56:40' 2024-09-14T06:56:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:56:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:56:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 06:56:40' 2024-09-14T06:56:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:56:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:56:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T06:56:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK42184267mirror.2smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589trueOK43434346mirror.1smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":2915970.23,"entries_behind_primary":0,"entries_per_second":182.33,"non_primary_position":{"entry_tid":1,"object_number":29,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":29,"tag_tid":6}}42104261mirror.0smithi1692024-09-14 06:56:40cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 06:56:40' 2024-09-14T06:56:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T06:56:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T06:56:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:56:40' 2024-09-14T06:56:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T06:56:51.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T06:56:51.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:56:51.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T06:56:51.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:56:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap3 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap3 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3 2024-09-14T06:56:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2024-09-14T06:56:52.709 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6789439392067834) > weight (0.25) 2024-09-14T06:56:52.710 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T06:56:52.710 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T06:56:52.710 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T06:56:52.711 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T06:56:52.708+0000 7fe9503e4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 40611) UID: 1000 2024-09-14T06:56:58.711 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T06:56:58.711 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:56:58.711 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T06:56:58.711 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T06:56:58.712 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T06:56:58.715 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T06:56:58.715 INFO:tasks.rbd_mirror_thrash:waiting for 105 secs before thrashing 2024-09-14T06:57:11.705 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5388478352584971) > weight (0.25) 2024-09-14T06:57:11.705 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T06:57:11.706 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T06:57:11.706 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T06:57:11.707 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T06:57:11.708+0000 7fcd73231640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 39942) UID: 1000 2024-09-14T06:57:17.707 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T06:57:17.707 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:57:17.707 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T06:57:17.707 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T06:57:17.708 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T06:57:17.711 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T06:57:17.711 INFO:tasks.rbd_mirror_thrash:waiting for 89 secs before thrashing 2024-09-14T06:57:22.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:22.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:57:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T06:57:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:57:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:57:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:57:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:23.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:57:23.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:57:23.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:57:23.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:57:23.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:23.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:23.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:23.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:23.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:57:23.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T06:57:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T06:57:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T06:57:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T06:57:23.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T06:57:23.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:23.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:23.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T06:57:23.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T06:57:23.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:23.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:23.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:23.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:23.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:23.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:23.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:23.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:23.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=37, tag_tid=8, entry_tid=8181], [object_number=36, tag_tid=8, entry_tid=8180], [object_number=39, tag_tid=8, entry_tid=8179], [object_number=38, tag_tid=8, entry_tid=8178]]], state=connected] 2024-09-14T06:57:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:23.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:23.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:23.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:23.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:23.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=37, tag_tid=8, entry_tid=8181], [object_number=36, tag_tid=8, entry_tid=8180], [object_number=39, tag_tid=8, entry_tid=8179], [object_number=38, tag_tid=8, entry_tid=8178]]], state=connected] 2024-09-14T06:57:23.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=37, tag_tid=8, entry_tid=8181' '!=' '' 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=37, tag_tid=8, entry_tid=8181' = '' 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:23.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:23.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:23.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:23.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:23.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:23.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:23.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:23.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:23.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:23.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8181 2024-09-14T06:57:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8181 2024-09-14T06:57:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:24.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:24.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:24.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:24.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:24.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:24.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:24.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:24.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:24.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=37, tag_tid=8, entry_tid=8289], [object_number=36, tag_tid=8, entry_tid=8288], [object_number=39, tag_tid=8, entry_tid=8287], [object_number=38, tag_tid=8, entry_tid=8286]]], state=connected] 2024-09-14T06:57:24.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:24.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:24.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:24.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:24.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:24.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=37, tag_tid=8, entry_tid=8289], [object_number=36, tag_tid=8, entry_tid=8288], [object_number=39, tag_tid=8, entry_tid=8287], [object_number=38, tag_tid=8, entry_tid=8286]]], state=connected] 2024-09-14T06:57:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=37, tag_tid=8, entry_tid=8289' '!=' 'object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=37, tag_tid=8, entry_tid=8289' = 'object_number=37, tag_tid=8, entry_tid=8181' 2024-09-14T06:57:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:24.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:24.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:24.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:24.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:24.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:24.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:24.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:24.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:24.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8289 2024-09-14T06:57:24.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8289 2024-09-14T06:57:24.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:24.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:24.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:24.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:24.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:24.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:24.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:24.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:24.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:24.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:24.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:24.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:24.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:24.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:24.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:24.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=39, tag_tid=8, entry_tid=8399], [object_number=38, tag_tid=8, entry_tid=8398], [object_number=37, tag_tid=8, entry_tid=8397], [object_number=36, tag_tid=8, entry_tid=8396]]], state=connected] 2024-09-14T06:57:24.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:24.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=39, tag_tid=8, entry_tid=8399], [object_number=38, tag_tid=8, entry_tid=8398], [object_number=37, tag_tid=8, entry_tid=8397], [object_number=36, tag_tid=8, entry_tid=8396]]], state=connected] 2024-09-14T06:57:24.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=39, tag_tid=8, entry_tid=8399' '!=' 'object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=39, tag_tid=8, entry_tid=8399' = 'object_number=37, tag_tid=8, entry_tid=8289' 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:24.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:24.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:24.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:24.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8399 2024-09-14T06:57:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8399 2024-09-14T06:57:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:25.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:25.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:25.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:25.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:25.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:25.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:25.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:25.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:25.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:25.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=39, tag_tid=8, entry_tid=8515], [object_number=38, tag_tid=8, entry_tid=8514], [object_number=37, tag_tid=8, entry_tid=8513], [object_number=36, tag_tid=8, entry_tid=8512]]], state=connected] 2024-09-14T06:57:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:25.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:25.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:25.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:25.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:25.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:25.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=39, tag_tid=8, entry_tid=8515], [object_number=38, tag_tid=8, entry_tid=8514], [object_number=37, tag_tid=8, entry_tid=8513], [object_number=36, tag_tid=8, entry_tid=8512]]], state=connected] 2024-09-14T06:57:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=39, tag_tid=8, entry_tid=8515' '!=' 'object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=39, tag_tid=8, entry_tid=8515' = 'object_number=39, tag_tid=8, entry_tid=8399' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:25.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:25.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8515 2024-09-14T06:57:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8515 2024-09-14T06:57:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:25.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:25.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:25.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:25.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:25.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:25.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:25.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:25.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:25.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=37, tag_tid=8, entry_tid=8633], [object_number=36, tag_tid=8, entry_tid=8632], [object_number=39, tag_tid=8, entry_tid=8631], [object_number=38, tag_tid=8, entry_tid=8630]]], state=connected] 2024-09-14T06:57:25.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:25.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:25.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=37, tag_tid=8, entry_tid=8633], [object_number=36, tag_tid=8, entry_tid=8632], [object_number=39, tag_tid=8, entry_tid=8631], [object_number=38, tag_tid=8, entry_tid=8630]]], state=connected] 2024-09-14T06:57:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=37, tag_tid=8, entry_tid=8633' '!=' 'object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=37, tag_tid=8, entry_tid=8633' = 'object_number=39, tag_tid=8, entry_tid=8515' 2024-09-14T06:57:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:25.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:25.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8633 2024-09-14T06:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8633 2024-09-14T06:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:26.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:26.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:26.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:26.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:26.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:26.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:26.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:26.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:26.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:26.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:26.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:26.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=36, tag_tid=8, entry_tid=8748], [object_number=39, tag_tid=8, entry_tid=8747], [object_number=38, tag_tid=8, entry_tid=8746], [object_number=37, tag_tid=8, entry_tid=8745]]], state=connected] 2024-09-14T06:57:26.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:26.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:26.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:26.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:26.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:26.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=36, tag_tid=8, entry_tid=8748], [object_number=39, tag_tid=8, entry_tid=8747], [object_number=38, tag_tid=8, entry_tid=8746], [object_number=37, tag_tid=8, entry_tid=8745]]], state=connected] 2024-09-14T06:57:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=36, tag_tid=8, entry_tid=8748' '!=' 'object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=36, tag_tid=8, entry_tid=8748' = 'object_number=37, tag_tid=8, entry_tid=8633' 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:26.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:26.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8748 2024-09-14T06:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8748 2024-09-14T06:57:26.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:26.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:26.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:26.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:26.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:26.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:26.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:26.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:26.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:26.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:26.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:26.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:26.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:26.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=39, tag_tid=8, entry_tid=8863], [object_number=38, tag_tid=8, entry_tid=8862], [object_number=37, tag_tid=8, entry_tid=8861], [object_number=36, tag_tid=8, entry_tid=8860]]], state=connected] 2024-09-14T06:57:26.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:26.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:26.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:26.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 9 2024-09-14T06:57:26.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:26.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:26.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:26.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=39, tag_tid=8, entry_tid=8863], [object_number=38, tag_tid=8, entry_tid=8862], [object_number=37, tag_tid=8, entry_tid=8861], [object_number=36, tag_tid=8, entry_tid=8860]]], state=connected] 2024-09-14T06:57:26.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=39, tag_tid=8, entry_tid=8863' '!=' 'object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=39, tag_tid=8, entry_tid=8863' = 'object_number=36, tag_tid=8, entry_tid=8748' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:26.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:26.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:26.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:26.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:26.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8863 2024-09-14T06:57:26.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8863 2024-09-14T06:57:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:27.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:27.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:27.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:27.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:27.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:27.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:27.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:27.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:27.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:27.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:27.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:27.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:27.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:27.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=8935], [object_number=42, tag_tid=8, entry_tid=8934], [object_number=41, tag_tid=8, entry_tid=8933], [object_number=40, tag_tid=8, entry_tid=8932]]], state=connected] 2024-09-14T06:57:27.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:27.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:27.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:27.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=8935], [object_number=42, tag_tid=8, entry_tid=8934], [object_number=41, tag_tid=8, entry_tid=8933], [object_number=40, tag_tid=8, entry_tid=8932]]], state=connected] 2024-09-14T06:57:27.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=8935' '!=' 'object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:27.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:27.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=8935' = 'object_number=39, tag_tid=8, entry_tid=8863' 2024-09-14T06:57:27.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:27.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:27.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:27.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:27.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:27.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:27.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:27.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:27.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:27.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8935 2024-09-14T06:57:27.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 8935 2024-09-14T06:57:27.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:27.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:27.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:27.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:27.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:27.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:27.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:27.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:27.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:27.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:27.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:27.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:27.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:27.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:27.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:27.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:27.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:27.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:27.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:27.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=9053], [object_number=40, tag_tid=8, entry_tid=9052], [object_number=43, tag_tid=8, entry_tid=9051], [object_number=42, tag_tid=8, entry_tid=9050]]], state=connected] 2024-09-14T06:57:27.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:27.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:27.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:27.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:27.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=9053], [object_number=40, tag_tid=8, entry_tid=9052], [object_number=43, tag_tid=8, entry_tid=9051], [object_number=42, tag_tid=8, entry_tid=9050]]], state=connected] 2024-09-14T06:57:27.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:27.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:27.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=9053' '!=' 'object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=9053' = 'object_number=43, tag_tid=8, entry_tid=8935' 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:27.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:27.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:27.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:27.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:27.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:27.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:27.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:27.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:27.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:27.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9053 2024-09-14T06:57:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9053 2024-09-14T06:57:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:28.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:28.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:28.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:28.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:28.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:28.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:28.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:28.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:28.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:28.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:28.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:28.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:28.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:28.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:28.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:28.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:28.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:28.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:28.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:28.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9164], [object_number=43, tag_tid=8, entry_tid=9163], [object_number=42, tag_tid=8, entry_tid=9162], [object_number=41, tag_tid=8, entry_tid=9161]]], state=connected] 2024-09-14T06:57:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:28.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:28.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:28.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:28.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9164], [object_number=43, tag_tid=8, entry_tid=9163], [object_number=42, tag_tid=8, entry_tid=9162], [object_number=41, tag_tid=8, entry_tid=9161]]], state=connected] 2024-09-14T06:57:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9164' '!=' 'object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9164' = 'object_number=41, tag_tid=8, entry_tid=9053' 2024-09-14T06:57:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:28.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:28.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9164 2024-09-14T06:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9164 2024-09-14T06:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:28.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:28.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:28.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:28.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:28.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:28.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:28.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:28.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:28.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:28.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=9270], [object_number=41, tag_tid=8, entry_tid=9269], [object_number=40, tag_tid=8, entry_tid=9268], [object_number=43, tag_tid=8, entry_tid=9267]]], state=connected] 2024-09-14T06:57:28.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=9270], [object_number=41, tag_tid=8, entry_tid=9269], [object_number=40, tag_tid=8, entry_tid=9268], [object_number=43, tag_tid=8, entry_tid=9267]]], state=connected] 2024-09-14T06:57:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=9270' '!=' 'object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=9270' = 'object_number=40, tag_tid=8, entry_tid=9164' 2024-09-14T06:57:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:28.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:28.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:28.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:28.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:28.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:28.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:28.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9270 2024-09-14T06:57:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9270 2024-09-14T06:57:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:29.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:29.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:29.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:29.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:29.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:29.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9364], [object_number=43, tag_tid=8, entry_tid=9363], [object_number=42, tag_tid=8, entry_tid=9362], [object_number=41, tag_tid=8, entry_tid=9361]]], state=connected] 2024-09-14T06:57:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:29.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:29.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:29.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:29.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:29.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:29.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:29.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:29.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:29.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9364], [object_number=43, tag_tid=8, entry_tid=9363], [object_number=42, tag_tid=8, entry_tid=9362], [object_number=41, tag_tid=8, entry_tid=9361]]], state=connected] 2024-09-14T06:57:29.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9364' '!=' 'object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9364' = 'object_number=42, tag_tid=8, entry_tid=9270' 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:29.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:29.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:29.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:29.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:29.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:29.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:29.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9364 2024-09-14T06:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9364 2024-09-14T06:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:29.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:29.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:29.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:29.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:29.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:29.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:29.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:29.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:29.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:29.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:29.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:29.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:29.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=9465], [object_number=40, tag_tid=8, entry_tid=9464], [object_number=43, tag_tid=8, entry_tid=9463], [object_number=42, tag_tid=8, entry_tid=9462]]], state=connected] 2024-09-14T06:57:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:29.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:29.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:29.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:29.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=9465], [object_number=40, tag_tid=8, entry_tid=9464], [object_number=43, tag_tid=8, entry_tid=9463], [object_number=42, tag_tid=8, entry_tid=9462]]], state=connected] 2024-09-14T06:57:29.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:29.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:29.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=9465' '!=' 'object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=9465' = 'object_number=40, tag_tid=8, entry_tid=9364' 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:29.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:29.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:29.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:29.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:29.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:29.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:29.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:29.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:29.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:29.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9465 2024-09-14T06:57:29.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9465 2024-09-14T06:57:29.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:29.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:29.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:30.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:30.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:30.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:30.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:30.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:30.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:30.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:30.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:30.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:30.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9588], [object_number=43, tag_tid=8, entry_tid=9587], [object_number=42, tag_tid=8, entry_tid=9586], [object_number=41, tag_tid=8, entry_tid=9585]]], state=connected] 2024-09-14T06:57:30.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:30.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:30.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:30.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:30.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:30.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:30.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:30.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9588], [object_number=43, tag_tid=8, entry_tid=9587], [object_number=42, tag_tid=8, entry_tid=9586], [object_number=41, tag_tid=8, entry_tid=9585]]], state=connected] 2024-09-14T06:57:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9588' '!=' 'object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9588' = 'object_number=41, tag_tid=8, entry_tid=9465' 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:30.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9588 2024-09-14T06:57:30.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9588 2024-09-14T06:57:30.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:30.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:30.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:30.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:30.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:30.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:30.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:30.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:30.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:30.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:30.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:30.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:30.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:30.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:30.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:30.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=9715], [object_number=42, tag_tid=8, entry_tid=9714], [object_number=41, tag_tid=8, entry_tid=9713], [object_number=40, tag_tid=8, entry_tid=9712]]], state=connected] 2024-09-14T06:57:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=9715], [object_number=42, tag_tid=8, entry_tid=9714], [object_number=41, tag_tid=8, entry_tid=9713], [object_number=40, tag_tid=8, entry_tid=9712]]], state=connected] 2024-09-14T06:57:30.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=9715' '!=' 'object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=9715' = 'object_number=40, tag_tid=8, entry_tid=9588' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9715 2024-09-14T06:57:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9715 2024-09-14T06:57:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:30.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:31.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:31.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:31.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:31.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:31.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9820], [object_number=43, tag_tid=8, entry_tid=9819], [object_number=42, tag_tid=8, entry_tid=9818], [object_number=41, tag_tid=8, entry_tid=9817]]], state=connected] 2024-09-14T06:57:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:31.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:31.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:31.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:31.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:31.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:31.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:31.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:31.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:31.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=9820], [object_number=43, tag_tid=8, entry_tid=9819], [object_number=42, tag_tid=8, entry_tid=9818], [object_number=41, tag_tid=8, entry_tid=9817]]], state=connected] 2024-09-14T06:57:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9820' '!=' 'object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=9820' = 'object_number=43, tag_tid=8, entry_tid=9715' 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:31.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:31.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:31.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9820 2024-09-14T06:57:31.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9820 2024-09-14T06:57:31.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:31.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:31.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:31.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:31.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:31.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:31.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:31.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:31.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:31.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:31.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:31.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:31.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:31.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:31.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:31.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:31.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:31.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:31.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:31.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:31.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:31.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:31.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:31.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:31.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:31.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:31.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:31.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=9923], [object_number=42, tag_tid=8, entry_tid=9922], [object_number=41, tag_tid=8, entry_tid=9921], [object_number=40, tag_tid=8, entry_tid=9920]]], state=connected] 2024-09-14T06:57:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:31.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:31.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=9923], [object_number=42, tag_tid=8, entry_tid=9922], [object_number=41, tag_tid=8, entry_tid=9921], [object_number=40, tag_tid=8, entry_tid=9920]]], state=connected] 2024-09-14T06:57:31.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=9923' '!=' 'object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=9923' = 'object_number=40, tag_tid=8, entry_tid=9820' 2024-09-14T06:57:31.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:31.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:31.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:31.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:31.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:31.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:31.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:31.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:31.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:31.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:31.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:31.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:31.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9923 2024-09-14T06:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 9923 2024-09-14T06:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:31.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:31.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:31.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:31.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:31.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:31.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:31.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:31.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:31.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:32.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:32.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:32.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10028], [object_number=43, tag_tid=8, entry_tid=10027], [object_number=42, tag_tid=8, entry_tid=10026], [object_number=41, tag_tid=8, entry_tid=10025]]], state=connected] 2024-09-14T06:57:32.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:32.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:32.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:32.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:32.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10028], [object_number=43, tag_tid=8, entry_tid=10027], [object_number=42, tag_tid=8, entry_tid=10026], [object_number=41, tag_tid=8, entry_tid=10025]]], state=connected] 2024-09-14T06:57:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10028' '!=' 'object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10028' = 'object_number=43, tag_tid=8, entry_tid=9923' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:32.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:32.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:32.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:32.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:32.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:32.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:32.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10028 2024-09-14T06:57:32.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10028 2024-09-14T06:57:32.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:32.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:32.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:32.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:32.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:32.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:32.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:32.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:32.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:32.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:32.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:32.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:32.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:32.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:32.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:32.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:32.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10138], [object_number=41, tag_tid=8, entry_tid=10137], [object_number=40, tag_tid=8, entry_tid=10136], [object_number=43, tag_tid=8, entry_tid=10135]]], state=connected] 2024-09-14T06:57:32.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:32.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:32.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:32.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:32.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:32.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10138], [object_number=41, tag_tid=8, entry_tid=10137], [object_number=40, tag_tid=8, entry_tid=10136], [object_number=43, tag_tid=8, entry_tid=10135]]], state=connected] 2024-09-14T06:57:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:32.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:32.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:32.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10138' '!=' 'object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:32.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10138' = 'object_number=40, tag_tid=8, entry_tid=10028' 2024-09-14T06:57:32.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:32.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:32.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:32.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:32.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:32.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:32.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:32.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:32.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:32.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:32.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:32.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:32.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:32.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10138 2024-09-14T06:57:32.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10138 2024-09-14T06:57:32.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:32.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:32.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:32.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:32.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:32.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:32.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:32.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:32.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:32.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:32.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:33.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:33.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:33.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:33.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:33.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=10203], [object_number=42, tag_tid=8, entry_tid=10202], [object_number=41, tag_tid=8, entry_tid=10201], [object_number=40, tag_tid=8, entry_tid=10200]]], state=connected] 2024-09-14T06:57:33.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:33.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:33.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:33.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:33.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:33.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:33.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=10203], [object_number=42, tag_tid=8, entry_tid=10202], [object_number=41, tag_tid=8, entry_tid=10201], [object_number=40, tag_tid=8, entry_tid=10200]]], state=connected] 2024-09-14T06:57:33.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=10203' '!=' 'object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=10203' = 'object_number=42, tag_tid=8, entry_tid=10138' 2024-09-14T06:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:33.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:33.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:33.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:33.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:33.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:33.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:33.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10203 2024-09-14T06:57:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10203 2024-09-14T06:57:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:33.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:33.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:33.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:33.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:33.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:33.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:33.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:33.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:33.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:33.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:33.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:33.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:33.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10306], [object_number=41, tag_tid=8, entry_tid=10305], [object_number=40, tag_tid=8, entry_tid=10304], [object_number=43, tag_tid=8, entry_tid=10303]]], state=connected] 2024-09-14T06:57:33.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:33.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:33.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:33.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:33.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10306], [object_number=41, tag_tid=8, entry_tid=10305], [object_number=40, tag_tid=8, entry_tid=10304], [object_number=43, tag_tid=8, entry_tid=10303]]], state=connected] 2024-09-14T06:57:33.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10306' '!=' 'object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10306' = 'object_number=43, tag_tid=8, entry_tid=10203' 2024-09-14T06:57:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:33.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:33.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:33.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:33.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:33.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:33.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10306 2024-09-14T06:57:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10306 2024-09-14T06:57:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:33.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:33.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:33.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:33.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:33.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:33.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:33.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:33.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:33.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:33.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:33.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:34.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:34.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:34.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:34.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:34.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:34.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:34.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10412], [object_number=43, tag_tid=8, entry_tid=10411], [object_number=42, tag_tid=8, entry_tid=10410], [object_number=41, tag_tid=8, entry_tid=10409]]], state=connected] 2024-09-14T06:57:34.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:34.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:34.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10412], [object_number=43, tag_tid=8, entry_tid=10411], [object_number=42, tag_tid=8, entry_tid=10410], [object_number=41, tag_tid=8, entry_tid=10409]]], state=connected] 2024-09-14T06:57:34.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10412' '!=' 'object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10412' = 'object_number=42, tag_tid=8, entry_tid=10306' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:34.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:34.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:34.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:34.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:34.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10412 2024-09-14T06:57:34.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10412 2024-09-14T06:57:34.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:34.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:34.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:34.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:34.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:34.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:34.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:34.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:34.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:34.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:34.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:34.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=10527], [object_number=42, tag_tid=8, entry_tid=10526], [object_number=41, tag_tid=8, entry_tid=10525], [object_number=40, tag_tid=8, entry_tid=10524]]], state=connected] 2024-09-14T06:57:34.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=10527], [object_number=42, tag_tid=8, entry_tid=10526], [object_number=41, tag_tid=8, entry_tid=10525], [object_number=40, tag_tid=8, entry_tid=10524]]], state=connected] 2024-09-14T06:57:34.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=10527' '!=' 'object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=10527' = 'object_number=40, tag_tid=8, entry_tid=10412' 2024-09-14T06:57:34.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:34.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:34.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:34.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:34.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:34.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:34.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10527 2024-09-14T06:57:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10527 2024-09-14T06:57:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:34.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:34.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:34.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:34.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:34.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:34.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:34.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:34.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:34.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:34.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:34.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:34.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:34.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:35.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:35.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:35.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:35.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:35.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:35.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:35.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:35.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10636], [object_number=43, tag_tid=8, entry_tid=10635], [object_number=42, tag_tid=8, entry_tid=10634], [object_number=41, tag_tid=8, entry_tid=10633]]], state=connected] 2024-09-14T06:57:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:35.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:35.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:35.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:35.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10636], [object_number=43, tag_tid=8, entry_tid=10635], [object_number=42, tag_tid=8, entry_tid=10634], [object_number=41, tag_tid=8, entry_tid=10633]]], state=connected] 2024-09-14T06:57:35.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10636' '!=' 'object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10636' = 'object_number=43, tag_tid=8, entry_tid=10527' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:35.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10636 2024-09-14T06:57:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10636 2024-09-14T06:57:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:35.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:35.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:35.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:35.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:35.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:35.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:35.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:35.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:35.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:35.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:35.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:35.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:35.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:35.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:35.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:35.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10744], [object_number=43, tag_tid=8, entry_tid=10743], [object_number=42, tag_tid=8, entry_tid=10742], [object_number=41, tag_tid=8, entry_tid=10741]]], state=connected] 2024-09-14T06:57:35.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:35.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:35.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=10744], [object_number=43, tag_tid=8, entry_tid=10743], [object_number=42, tag_tid=8, entry_tid=10742], [object_number=41, tag_tid=8, entry_tid=10741]]], state=connected] 2024-09-14T06:57:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10744' '!=' 'object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=10744' = 'object_number=40, tag_tid=8, entry_tid=10636' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:35.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:35.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:35.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:35.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:35.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:35.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:35.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:35.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10744 2024-09-14T06:57:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10744 2024-09-14T06:57:35.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:35.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:35.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:35.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:35.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:35.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:35.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:35.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:35.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:35.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:35.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:35.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:35.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:35.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:36.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10858], [object_number=41, tag_tid=8, entry_tid=10857], [object_number=40, tag_tid=8, entry_tid=10856], [object_number=43, tag_tid=8, entry_tid=10855]]], state=connected] 2024-09-14T06:57:36.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:36.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:36.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:36.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:36.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:36.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:36.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10858], [object_number=41, tag_tid=8, entry_tid=10857], [object_number=40, tag_tid=8, entry_tid=10856], [object_number=43, tag_tid=8, entry_tid=10855]]], state=connected] 2024-09-14T06:57:36.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10858' '!=' 'object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10858' = 'object_number=40, tag_tid=8, entry_tid=10744' 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:36.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:36.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:36.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10858 2024-09-14T06:57:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10858 2024-09-14T06:57:36.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:36.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:36.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:36.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:36.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:36.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:36.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:36.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:36.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:36.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:36.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:36.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:36.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10958], [object_number=41, tag_tid=8, entry_tid=10957], [object_number=40, tag_tid=8, entry_tid=10956], [object_number=43, tag_tid=8, entry_tid=10955]]], state=connected] 2024-09-14T06:57:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:36.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:36.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:36.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:36.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:36.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:36.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=10958], [object_number=41, tag_tid=8, entry_tid=10957], [object_number=40, tag_tid=8, entry_tid=10956], [object_number=43, tag_tid=8, entry_tid=10955]]], state=connected] 2024-09-14T06:57:36.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10958' '!=' 'object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=10958' = 'object_number=42, tag_tid=8, entry_tid=10858' 2024-09-14T06:57:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:36.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:36.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:36.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:36.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:36.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:36.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:36.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:36.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:36.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:36.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:36.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:36.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:36.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10958 2024-09-14T06:57:36.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 10958 2024-09-14T06:57:36.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:36.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:36.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:36.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:36.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:36.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:36.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:36.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:36.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:36.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:36.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:36.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:36.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:36.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:36.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:36.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:36.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:36.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:36.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:36.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:36.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:36.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:36.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:36.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:37.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:37.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:37.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=11061], [object_number=40, tag_tid=8, entry_tid=11060], [object_number=43, tag_tid=8, entry_tid=11059], [object_number=42, tag_tid=8, entry_tid=11058]]], state=connected] 2024-09-14T06:57:37.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:37.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:37.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:37.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:37.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:37.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:37.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:37.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:37.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=11061], [object_number=40, tag_tid=8, entry_tid=11060], [object_number=43, tag_tid=8, entry_tid=11059], [object_number=42, tag_tid=8, entry_tid=11058]]], state=connected] 2024-09-14T06:57:37.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=11061' '!=' 'object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=11061' = 'object_number=42, tag_tid=8, entry_tid=10958' 2024-09-14T06:57:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:37.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:37.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:37.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:37.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:37.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:37.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:37.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11061 2024-09-14T06:57:37.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11061 2024-09-14T06:57:37.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:37.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:37.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:37.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:37.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:37.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:37.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:37.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:37.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:37.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:37.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:37.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:37.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11180], [object_number=43, tag_tid=8, entry_tid=11179], [object_number=42, tag_tid=8, entry_tid=11178], [object_number=41, tag_tid=8, entry_tid=11177]]], state=connected] 2024-09-14T06:57:37.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11180], [object_number=43, tag_tid=8, entry_tid=11179], [object_number=42, tag_tid=8, entry_tid=11178], [object_number=41, tag_tid=8, entry_tid=11177]]], state=connected] 2024-09-14T06:57:37.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11180' '!=' 'object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11180' = 'object_number=41, tag_tid=8, entry_tid=11061' 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:37.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:37.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:37.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:37.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:37.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11180 2024-09-14T06:57:37.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11180 2024-09-14T06:57:37.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:37.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:37.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:37.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:37.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:37.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:37.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:37.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:37.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:37.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:37.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:37.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:37.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:37.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:37.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:37.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:37.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:37.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:37.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:37.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:37.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:37.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:37.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:37.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:37.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:37.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:38.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:38.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:38.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:38.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=11289], [object_number=40, tag_tid=8, entry_tid=11288], [object_number=43, tag_tid=8, entry_tid=11287], [object_number=42, tag_tid=8, entry_tid=11286]]], state=connected] 2024-09-14T06:57:38.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:38.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:38.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=11289], [object_number=40, tag_tid=8, entry_tid=11288], [object_number=43, tag_tid=8, entry_tid=11287], [object_number=42, tag_tid=8, entry_tid=11286]]], state=connected] 2024-09-14T06:57:38.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=11289' '!=' 'object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=11289' = 'object_number=40, tag_tid=8, entry_tid=11180' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:38.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:38.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:38.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:38.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:38.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:38.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:38.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11289 2024-09-14T06:57:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11289 2024-09-14T06:57:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:38.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:38.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:38.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:38.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:38.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:38.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:38.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:38.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:38.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:38.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:38.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:38.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=11390], [object_number=41, tag_tid=8, entry_tid=11389], [object_number=40, tag_tid=8, entry_tid=11388], [object_number=43, tag_tid=8, entry_tid=11387]]], state=connected] 2024-09-14T06:57:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=11390], [object_number=41, tag_tid=8, entry_tid=11389], [object_number=40, tag_tid=8, entry_tid=11388], [object_number=43, tag_tid=8, entry_tid=11387]]], state=connected] 2024-09-14T06:57:38.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:38.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=11390' '!=' 'object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=11390' = 'object_number=41, tag_tid=8, entry_tid=11289' 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:38.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:38.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:38.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:38.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:38.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:38.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:38.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11390 2024-09-14T06:57:38.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11390 2024-09-14T06:57:38.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:38.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:38.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:38.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:38.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:38.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:38.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:38.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:38.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:38.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:38.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=11498], [object_number=41, tag_tid=8, entry_tid=11497], [object_number=40, tag_tid=8, entry_tid=11496], [object_number=43, tag_tid=8, entry_tid=11495]]], state=connected] 2024-09-14T06:57:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:39.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:39.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:39.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:39.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=11498], [object_number=41, tag_tid=8, entry_tid=11497], [object_number=40, tag_tid=8, entry_tid=11496], [object_number=43, tag_tid=8, entry_tid=11495]]], state=connected] 2024-09-14T06:57:39.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=11498' '!=' 'object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=11498' = 'object_number=42, tag_tid=8, entry_tid=11390' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:39.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:39.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:39.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11498 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11498 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:39.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:39.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:39.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:39.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:39.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:39.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:39.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:39.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:39.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:39.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:39.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=11609], [object_number=40, tag_tid=8, entry_tid=11608], [object_number=43, tag_tid=8, entry_tid=11607], [object_number=42, tag_tid=8, entry_tid=11606]]], state=connected] 2024-09-14T06:57:39.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:39.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:39.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:39.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:39.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:39.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=11609], [object_number=40, tag_tid=8, entry_tid=11608], [object_number=43, tag_tid=8, entry_tid=11607], [object_number=42, tag_tid=8, entry_tid=11606]]], state=connected] 2024-09-14T06:57:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:39.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=11609' '!=' 'object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=11609' = 'object_number=42, tag_tid=8, entry_tid=11498' 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:39.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:39.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:39.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:39.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:39.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:39.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:39.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:39.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11609 2024-09-14T06:57:39.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11609 2024-09-14T06:57:39.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:39.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:39.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:39.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:39.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:39.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:39.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:39.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:39.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11716], [object_number=43, tag_tid=8, entry_tid=11715], [object_number=42, tag_tid=8, entry_tid=11714], [object_number=41, tag_tid=8, entry_tid=11713]]], state=connected] 2024-09-14T06:57:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:40.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11716], [object_number=43, tag_tid=8, entry_tid=11715], [object_number=42, tag_tid=8, entry_tid=11714], [object_number=41, tag_tid=8, entry_tid=11713]]], state=connected] 2024-09-14T06:57:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11716' '!=' 'object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11716' = 'object_number=41, tag_tid=8, entry_tid=11609' 2024-09-14T06:57:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:40.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11716 2024-09-14T06:57:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11716 2024-09-14T06:57:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:40.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:40.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:40.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:40.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:40.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:40.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:40.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:40.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:40.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11828], [object_number=43, tag_tid=8, entry_tid=11827], [object_number=42, tag_tid=8, entry_tid=11826], [object_number=41, tag_tid=8, entry_tid=11825]]], state=connected] 2024-09-14T06:57:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:40.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:40.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:40.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:40.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:40.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:40.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:40.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11828], [object_number=43, tag_tid=8, entry_tid=11827], [object_number=42, tag_tid=8, entry_tid=11826], [object_number=41, tag_tid=8, entry_tid=11825]]], state=connected] 2024-09-14T06:57:40.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:40.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11828' '!=' 'object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11828' = 'object_number=40, tag_tid=8, entry_tid=11716' 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:40.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:40.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:40.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11828 2024-09-14T06:57:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11828 2024-09-14T06:57:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:40.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:40.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:40.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:40.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:40.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:40.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:40.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:40.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:40.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:40.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:40.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:40.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:40.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:40.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:40.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:40.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11948], [object_number=43, tag_tid=8, entry_tid=11947], [object_number=42, tag_tid=8, entry_tid=11946], [object_number=41, tag_tid=8, entry_tid=11945]]], state=connected] 2024-09-14T06:57:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=11948], [object_number=43, tag_tid=8, entry_tid=11947], [object_number=42, tag_tid=8, entry_tid=11946], [object_number=41, tag_tid=8, entry_tid=11945]]], state=connected] 2024-09-14T06:57:41.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11948' '!=' 'object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=11948' = 'object_number=40, tag_tid=8, entry_tid=11828' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:41.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:41.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:41.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:41.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11948 2024-09-14T06:57:41.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 11948 2024-09-14T06:57:41.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:41.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:41.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:41.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:41.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:41.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:41.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:41.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:41.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:41.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:41.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:41.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:41.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:41.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:41.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:41.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:41.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:41.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:41.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=12067], [object_number=42, tag_tid=8, entry_tid=12066], [object_number=41, tag_tid=8, entry_tid=12065], [object_number=40, tag_tid=8, entry_tid=12064]]], state=connected] 2024-09-14T06:57:41.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:41.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:41.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:41.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:41.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=12067], [object_number=42, tag_tid=8, entry_tid=12066], [object_number=41, tag_tid=8, entry_tid=12065], [object_number=40, tag_tid=8, entry_tid=12064]]], state=connected] 2024-09-14T06:57:41.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=12067' '!=' 'object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=12067' = 'object_number=40, tag_tid=8, entry_tid=11948' 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:41.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:41.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:41.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:41.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:41.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12067 2024-09-14T06:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12067 2024-09-14T06:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:41.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:41.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:41.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:41.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:41.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:41.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:41.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:41.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:41.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:41.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=12179], [object_number=42, tag_tid=8, entry_tid=12178], [object_number=41, tag_tid=8, entry_tid=12177], [object_number=40, tag_tid=8, entry_tid=12176]]], state=connected] 2024-09-14T06:57:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:41.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:41.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:41.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:41.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:41.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=12179], [object_number=42, tag_tid=8, entry_tid=12178], [object_number=41, tag_tid=8, entry_tid=12177], [object_number=40, tag_tid=8, entry_tid=12176]]], state=connected] 2024-09-14T06:57:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=12179' '!=' 'object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=12179' = 'object_number=43, tag_tid=8, entry_tid=12067' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:41.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:41.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:41.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:41.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:41.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:41.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:41.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:41.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:41.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:41.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12179 2024-09-14T06:57:42.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12179 2024-09-14T06:57:42.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:42.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:42.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:42.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:42.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:42.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:42.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:42.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:42.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:42.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:42.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:42.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:42.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:42.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:42.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:42.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:42.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:42.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:42.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=12289], [object_number=40, tag_tid=8, entry_tid=12288], [object_number=43, tag_tid=8, entry_tid=12287], [object_number=42, tag_tid=8, entry_tid=12286]]], state=connected] 2024-09-14T06:57:42.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:42.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:42.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:42.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:42.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=12289], [object_number=40, tag_tid=8, entry_tid=12288], [object_number=43, tag_tid=8, entry_tid=12287], [object_number=42, tag_tid=8, entry_tid=12286]]], state=connected] 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=12289' '!=' 'object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=12289' = 'object_number=43, tag_tid=8, entry_tid=12179' 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:42.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12289 2024-09-14T06:57:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12289 2024-09-14T06:57:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:42.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:42.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:42.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:42.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:42.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:42.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:42.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:42.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:42.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:42.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:42.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:42.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=12389], [object_number=40, tag_tid=8, entry_tid=12388], [object_number=43, tag_tid=8, entry_tid=12387], [object_number=42, tag_tid=8, entry_tid=12386]]], state=connected] 2024-09-14T06:57:42.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:42.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:42.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:42.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:42.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:42.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=12389], [object_number=40, tag_tid=8, entry_tid=12388], [object_number=43, tag_tid=8, entry_tid=12387], [object_number=42, tag_tid=8, entry_tid=12386]]], state=connected] 2024-09-14T06:57:42.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=12389' '!=' 'object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=12389' = 'object_number=41, tag_tid=8, entry_tid=12289' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:42.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:42.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:42.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:42.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:42.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:42.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:42.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12389 2024-09-14T06:57:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12389 2024-09-14T06:57:42.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:42.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:42.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:43.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:43.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:43.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:43.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=12496], [object_number=43, tag_tid=8, entry_tid=12495], [object_number=42, tag_tid=8, entry_tid=12494], [object_number=41, tag_tid=8, entry_tid=12493]]], state=connected] 2024-09-14T06:57:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:43.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=12496], [object_number=43, tag_tid=8, entry_tid=12495], [object_number=42, tag_tid=8, entry_tid=12494], [object_number=41, tag_tid=8, entry_tid=12493]]], state=connected] 2024-09-14T06:57:43.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=12496' '!=' 'object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=12496' = 'object_number=41, tag_tid=8, entry_tid=12389' 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:43.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:43.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:43.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:43.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:43.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:43.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:43.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12496 2024-09-14T06:57:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12496 2024-09-14T06:57:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:43.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:43.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:43.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:43.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:43.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:43.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:43.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:43.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=12593], [object_number=40, tag_tid=8, entry_tid=12592], [object_number=43, tag_tid=8, entry_tid=12591], [object_number=42, tag_tid=8, entry_tid=12590]]], state=connected] 2024-09-14T06:57:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=41, tag_tid=8, entry_tid=12593], [object_number=40, tag_tid=8, entry_tid=12592], [object_number=43, tag_tid=8, entry_tid=12591], [object_number=42, tag_tid=8, entry_tid=12590]]], state=connected] 2024-09-14T06:57:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=12593' '!=' 'object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=41, tag_tid=8, entry_tid=12593' = 'object_number=40, tag_tid=8, entry_tid=12496' 2024-09-14T06:57:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:43.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:43.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:43.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:43.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:43.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:43.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:43.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:43.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12593 2024-09-14T06:57:43.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12593 2024-09-14T06:57:43.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:43.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:43.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:44.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:44.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:44.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:44.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:44.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:44.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:44.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:44.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:44.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:44.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:44.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:44.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:44.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:44.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:44.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:44.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:44.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=12692], [object_number=43, tag_tid=8, entry_tid=12691], [object_number=42, tag_tid=8, entry_tid=12690], [object_number=41, tag_tid=8, entry_tid=12689]]], state=connected] 2024-09-14T06:57:44.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:44.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:44.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=40, tag_tid=8, entry_tid=12692], [object_number=43, tag_tid=8, entry_tid=12691], [object_number=42, tag_tid=8, entry_tid=12690], [object_number=41, tag_tid=8, entry_tid=12689]]], state=connected] 2024-09-14T06:57:44.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=12692' '!=' 'object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=40, tag_tid=8, entry_tid=12692' = 'object_number=41, tag_tid=8, entry_tid=12593' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:44.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:44.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12692 2024-09-14T06:57:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12692 2024-09-14T06:57:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:44.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:44.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:44.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=12799], [object_number=42, tag_tid=8, entry_tid=12798], [object_number=41, tag_tid=8, entry_tid=12797], [object_number=40, tag_tid=8, entry_tid=12796]]], state=connected] 2024-09-14T06:57:44.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:44.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:44.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:44.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:44.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:44.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:44.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:44.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:44.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:44.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:44.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:44.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=43, tag_tid=8, entry_tid=12799], [object_number=42, tag_tid=8, entry_tid=12798], [object_number=41, tag_tid=8, entry_tid=12797], [object_number=40, tag_tid=8, entry_tid=12796]]], state=connected] 2024-09-14T06:57:44.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:44.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:44.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:44.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=12799' '!=' 'object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:44.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=43, tag_tid=8, entry_tid=12799' = 'object_number=40, tag_tid=8, entry_tid=12692' 2024-09-14T06:57:44.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:44.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:44.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:44.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:44.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:44.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:44.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12799 2024-09-14T06:57:44.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12799 2024-09-14T06:57:44.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:44.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:44.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:45.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:45.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:45.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:45.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:45.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:45.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:45.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:45.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:45.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:45.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:45.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:45.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:45.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:45.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:45.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:45.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:45.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:45.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:45.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:45.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:45.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:45.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=12914], [object_number=41, tag_tid=8, entry_tid=12913], [object_number=40, tag_tid=8, entry_tid=12912], [object_number=43, tag_tid=8, entry_tid=12911]]], state=connected] 2024-09-14T06:57:45.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:45.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:45.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:45.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=42, tag_tid=8, entry_tid=12914], [object_number=41, tag_tid=8, entry_tid=12913], [object_number=40, tag_tid=8, entry_tid=12912], [object_number=43, tag_tid=8, entry_tid=12911]]], state=connected] 2024-09-14T06:57:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=12914' '!=' 'object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=42, tag_tid=8, entry_tid=12914' = 'object_number=43, tag_tid=8, entry_tid=12799' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:45.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:45.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:45.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:45.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:45.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:45.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12914 2024-09-14T06:57:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 12914 2024-09-14T06:57:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:45.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:45.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:45.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:45.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:45.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:45.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:45.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:45.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:45.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:45.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:45.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:45.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:45.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:45.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13017], [object_number=40, tag_tid=8, entry_tid=13016], [object_number=43, tag_tid=8, entry_tid=13015], [object_number=42, tag_tid=8, entry_tid=13014]]], state=connected] 2024-09-14T06:57:45.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 10 2024-09-14T06:57:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13017], [object_number=40, tag_tid=8, entry_tid=13016], [object_number=43, tag_tid=8, entry_tid=13015], [object_number=42, tag_tid=8, entry_tid=13014]]], state=connected] 2024-09-14T06:57:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13017' '!=' 'object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13017' = 'object_number=42, tag_tid=8, entry_tid=12914' 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:45.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:45.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:45.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13017 2024-09-14T06:57:45.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13017 2024-09-14T06:57:45.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:45.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:45.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:46.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:46.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:46.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:46.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:46.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:46.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:46.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:46.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:46.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:46.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:46.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:46.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:46.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:46.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:46.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:46.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:46.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=13118], [object_number=45, tag_tid=8, entry_tid=13117], [object_number=44, tag_tid=8, entry_tid=13116], [object_number=47, tag_tid=8, entry_tid=13115]]], state=connected] 2024-09-14T06:57:46.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:46.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:46.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:46.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:46.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=13118], [object_number=45, tag_tid=8, entry_tid=13117], [object_number=44, tag_tid=8, entry_tid=13116], [object_number=47, tag_tid=8, entry_tid=13115]]], state=connected] 2024-09-14T06:57:46.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=13118' '!=' 'object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=13118' = 'object_number=45, tag_tid=8, entry_tid=13017' 2024-09-14T06:57:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:46.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:46.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13118 2024-09-14T06:57:46.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13118 2024-09-14T06:57:46.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:46.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:46.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:46.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:46.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:46.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:46.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:46.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:46.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:46.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:46.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:46.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:46.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:46.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:46.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:46.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:46.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:46.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13233], [object_number=44, tag_tid=8, entry_tid=13232], [object_number=47, tag_tid=8, entry_tid=13231], [object_number=46, tag_tid=8, entry_tid=13230]]], state=connected] 2024-09-14T06:57:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:46.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:46.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:46.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:46.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:46.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:46.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:46.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:46.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13233], [object_number=44, tag_tid=8, entry_tid=13232], [object_number=47, tag_tid=8, entry_tid=13231], [object_number=46, tag_tid=8, entry_tid=13230]]], state=connected] 2024-09-14T06:57:46.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13233' '!=' 'object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13233' = 'object_number=46, tag_tid=8, entry_tid=13118' 2024-09-14T06:57:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:46.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:46.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:46.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:46.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:46.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:46.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:46.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:46.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13233 2024-09-14T06:57:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13233 2024-09-14T06:57:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:47.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:47.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:47.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:47.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:47.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:47.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:47.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:47.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:47.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:47.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:47.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:47.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:47.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:47.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:47.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:47.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=13352], [object_number=47, tag_tid=8, entry_tid=13351], [object_number=46, tag_tid=8, entry_tid=13350], [object_number=45, tag_tid=8, entry_tid=13349]]], state=connected] 2024-09-14T06:57:47.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:47.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:47.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:47.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:47.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:47.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:47.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=13352], [object_number=47, tag_tid=8, entry_tid=13351], [object_number=46, tag_tid=8, entry_tid=13350], [object_number=45, tag_tid=8, entry_tid=13349]]], state=connected] 2024-09-14T06:57:47.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=13352' '!=' 'object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=13352' = 'object_number=45, tag_tid=8, entry_tid=13233' 2024-09-14T06:57:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:47.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:47.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:47.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:47.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:47.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:47.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:47.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:47.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:47.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13352 2024-09-14T06:57:47.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13352 2024-09-14T06:57:47.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:47.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:47.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:47.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:47.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:47.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:47.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:47.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:47.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:47.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:47.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:47.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:47.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:47.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=13480], [object_number=47, tag_tid=8, entry_tid=13479], [object_number=46, tag_tid=8, entry_tid=13478], [object_number=45, tag_tid=8, entry_tid=13477]]], state=connected] 2024-09-14T06:57:47.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:47.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:47.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:47.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:47.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:47.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:47.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=13480], [object_number=47, tag_tid=8, entry_tid=13479], [object_number=46, tag_tid=8, entry_tid=13478], [object_number=45, tag_tid=8, entry_tid=13477]]], state=connected] 2024-09-14T06:57:47.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:47.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:47.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:47.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=13480' '!=' 'object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:47.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=13480' = 'object_number=44, tag_tid=8, entry_tid=13352' 2024-09-14T06:57:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:47.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:47.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:47.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:47.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:47.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:47.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:47.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:47.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13480 2024-09-14T06:57:47.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13480 2024-09-14T06:57:47.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:47.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:47.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:48.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:48.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:48.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:48.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:48.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:48.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:48.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:48.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:48.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:48.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:48.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:48.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:48.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:48.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=13603], [object_number=46, tag_tid=8, entry_tid=13602], [object_number=45, tag_tid=8, entry_tid=13601], [object_number=44, tag_tid=8, entry_tid=13600]]], state=connected] 2024-09-14T06:57:48.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:48.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:48.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:48.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:48.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:48.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:48.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:48.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:48.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:48.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:48.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:48.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:48.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=13603], [object_number=46, tag_tid=8, entry_tid=13602], [object_number=45, tag_tid=8, entry_tid=13601], [object_number=44, tag_tid=8, entry_tid=13600]]], state=connected] 2024-09-14T06:57:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=13603' '!=' 'object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=13603' = 'object_number=44, tag_tid=8, entry_tid=13480' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:48.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13603 2024-09-14T06:57:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13603 2024-09-14T06:57:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:48.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:48.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:48.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:48.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:48.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:48.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:48.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:48.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:48.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:48.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:48.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:48.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:48.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:48.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:48.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:48.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13729], [object_number=44, tag_tid=8, entry_tid=13728], [object_number=47, tag_tid=8, entry_tid=13727], [object_number=46, tag_tid=8, entry_tid=13726]]], state=connected] 2024-09-14T06:57:48.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13729], [object_number=44, tag_tid=8, entry_tid=13728], [object_number=47, tag_tid=8, entry_tid=13727], [object_number=46, tag_tid=8, entry_tid=13726]]], state=connected] 2024-09-14T06:57:48.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:48.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:48.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:48.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13729' '!=' 'object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:48.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13729' = 'object_number=47, tag_tid=8, entry_tid=13603' 2024-09-14T06:57:48.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:48.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:48.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:48.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:48.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:48.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:48.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:48.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:48.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:48.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:48.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:48.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13729 2024-09-14T06:57:48.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13729 2024-09-14T06:57:48.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:48.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:48.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:49.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:49.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:49.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:49.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:49.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:49.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:49.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:49.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:49.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:49.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13837], [object_number=44, tag_tid=8, entry_tid=13836], [object_number=47, tag_tid=8, entry_tid=13835], [object_number=46, tag_tid=8, entry_tid=13834]]], state=connected] 2024-09-14T06:57:49.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:49.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:49.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:49.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:49.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:49.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13837], [object_number=44, tag_tid=8, entry_tid=13836], [object_number=47, tag_tid=8, entry_tid=13835], [object_number=46, tag_tid=8, entry_tid=13834]]], state=connected] 2024-09-14T06:57:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13837' '!=' 'object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13837' = 'object_number=45, tag_tid=8, entry_tid=13729' 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:49.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:49.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:49.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:49.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:49.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:49.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:49.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:49.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:49.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13837 2024-09-14T06:57:49.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13837 2024-09-14T06:57:49.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:49.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:49.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:49.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:49.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:49.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:49.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:49.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:49.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:49.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:49.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:49.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:49.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:49.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:49.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:49.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:49.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:49.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:49.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:49.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:49.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:49.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:49.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:49.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13941], [object_number=44, tag_tid=8, entry_tid=13940], [object_number=47, tag_tid=8, entry_tid=13939], [object_number=46, tag_tid=8, entry_tid=13938]]], state=connected] 2024-09-14T06:57:49.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:49.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:49.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:49.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=13941], [object_number=44, tag_tid=8, entry_tid=13940], [object_number=47, tag_tid=8, entry_tid=13939], [object_number=46, tag_tid=8, entry_tid=13938]]], state=connected] 2024-09-14T06:57:49.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13941' '!=' 'object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=13941' = 'object_number=45, tag_tid=8, entry_tid=13837' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:49.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:49.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:49.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:49.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:49.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:49.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:49.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:49.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:49.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13941 2024-09-14T06:57:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 13941 2024-09-14T06:57:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:50.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:50.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:50.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:50.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:50.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:50.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:50.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:50.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:50.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:50.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:50.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:50.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:50.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:50.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:50.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:50.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:50.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:50.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:50.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:50.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:50.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:50.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:50.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=14051], [object_number=46, tag_tid=8, entry_tid=14050], [object_number=45, tag_tid=8, entry_tid=14049], [object_number=44, tag_tid=8, entry_tid=14048]]], state=connected] 2024-09-14T06:57:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:50.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:50.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:50.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:50.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:50.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:50.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=14051], [object_number=46, tag_tid=8, entry_tid=14050], [object_number=45, tag_tid=8, entry_tid=14049], [object_number=44, tag_tid=8, entry_tid=14048]]], state=connected] 2024-09-14T06:57:50.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=14051' '!=' 'object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:50.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:50.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=14051' = 'object_number=45, tag_tid=8, entry_tid=13941' 2024-09-14T06:57:50.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:50.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:50.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14051 2024-09-14T06:57:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14051 2024-09-14T06:57:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:50.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:50.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:50.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:50.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:50.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:50.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:50.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:50.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:50.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:50.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:50.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:50.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:50.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:50.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:50.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:50.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:50.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:50.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:50.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=14166], [object_number=45, tag_tid=8, entry_tid=14165], [object_number=44, tag_tid=8, entry_tid=14164], [object_number=47, tag_tid=8, entry_tid=14163]]], state=connected] 2024-09-14T06:57:50.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:50.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:50.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:50.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=14166], [object_number=45, tag_tid=8, entry_tid=14165], [object_number=44, tag_tid=8, entry_tid=14164], [object_number=47, tag_tid=8, entry_tid=14163]]], state=connected] 2024-09-14T06:57:50.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=14166' '!=' 'object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=14166' = 'object_number=47, tag_tid=8, entry_tid=14051' 2024-09-14T06:57:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:50.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:50.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:50.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:50.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:50.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:50.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:50.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:50.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14166 2024-09-14T06:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14166 2024-09-14T06:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:50.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:51.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:51.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:51.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:51.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:51.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:51.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:51.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:51.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:51.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14280], [object_number=47, tag_tid=8, entry_tid=14279], [object_number=46, tag_tid=8, entry_tid=14278], [object_number=45, tag_tid=8, entry_tid=14277]]], state=connected] 2024-09-14T06:57:51.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:51.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14280], [object_number=47, tag_tid=8, entry_tid=14279], [object_number=46, tag_tid=8, entry_tid=14278], [object_number=45, tag_tid=8, entry_tid=14277]]], state=connected] 2024-09-14T06:57:51.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14280' '!=' 'object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:51.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:51.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14280' = 'object_number=46, tag_tid=8, entry_tid=14166' 2024-09-14T06:57:51.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:51.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:51.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14280 2024-09-14T06:57:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14280 2024-09-14T06:57:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:51.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:51.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:51.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:51.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:51.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:51.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:51.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:51.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:51.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14396], [object_number=47, tag_tid=8, entry_tid=14395], [object_number=46, tag_tid=8, entry_tid=14394], [object_number=45, tag_tid=8, entry_tid=14393]]], state=connected] 2024-09-14T06:57:51.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:51.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:51.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:51.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:51.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:51.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:51.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:51.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14396], [object_number=47, tag_tid=8, entry_tid=14395], [object_number=46, tag_tid=8, entry_tid=14394], [object_number=45, tag_tid=8, entry_tid=14393]]], state=connected] 2024-09-14T06:57:51.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:51.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:51.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:51.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14396' '!=' 'object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14396' = 'object_number=44, tag_tid=8, entry_tid=14280' 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:51.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:51.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:51.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14396 2024-09-14T06:57:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14396 2024-09-14T06:57:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:52.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:52.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:52.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14520], [object_number=47, tag_tid=8, entry_tid=14519], [object_number=46, tag_tid=8, entry_tid=14518], [object_number=45, tag_tid=8, entry_tid=14517]]], state=connected] 2024-09-14T06:57:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:52.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:52.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:52.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:52.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:52.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14520], [object_number=47, tag_tid=8, entry_tid=14519], [object_number=46, tag_tid=8, entry_tid=14518], [object_number=45, tag_tid=8, entry_tid=14517]]], state=connected] 2024-09-14T06:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14520' '!=' 'object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14520' = 'object_number=44, tag_tid=8, entry_tid=14396' 2024-09-14T06:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:52.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:52.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:52.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:52.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:52.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14520 2024-09-14T06:57:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14520 2024-09-14T06:57:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:52.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:52.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:52.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:52.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:52.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:52.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:52.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:52.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:52.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:52.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:52.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:52.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:52.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:52.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:52.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:52.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=14630], [object_number=45, tag_tid=8, entry_tid=14629], [object_number=44, tag_tid=8, entry_tid=14628], [object_number=47, tag_tid=8, entry_tid=14627]]], state=connected] 2024-09-14T06:57:52.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:52.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=14630], [object_number=45, tag_tid=8, entry_tid=14629], [object_number=44, tag_tid=8, entry_tid=14628], [object_number=47, tag_tid=8, entry_tid=14627]]], state=connected] 2024-09-14T06:57:52.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=14630' '!=' 'object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=14630' = 'object_number=44, tag_tid=8, entry_tid=14520' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:52.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:52.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:52.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:52.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:52.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:52.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:52.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:52.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:52.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:52.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14630 2024-09-14T06:57:52.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14630 2024-09-14T06:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:53.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:53.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:53.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:53.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:53.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:53.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:53.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:53.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:53.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:53.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:53.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:53.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:53.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:53.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=14783], [object_number=46, tag_tid=8, entry_tid=14782], [object_number=45, tag_tid=8, entry_tid=14781], [object_number=44, tag_tid=8, entry_tid=14780]]], state=connected] 2024-09-14T06:57:53.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=14783], [object_number=46, tag_tid=8, entry_tid=14782], [object_number=45, tag_tid=8, entry_tid=14781], [object_number=44, tag_tid=8, entry_tid=14780]]], state=connected] 2024-09-14T06:57:53.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=14783' '!=' 'object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=14783' = 'object_number=46, tag_tid=8, entry_tid=14630' 2024-09-14T06:57:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:53.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:53.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:53.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14783 2024-09-14T06:57:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14783 2024-09-14T06:57:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:53.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:53.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:53.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:53.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:53.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:53.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:53.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:53.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:53.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:53.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:53.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:53.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:53.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:53.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:53.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14884], [object_number=47, tag_tid=8, entry_tid=14883], [object_number=46, tag_tid=8, entry_tid=14882], [object_number=45, tag_tid=8, entry_tid=14881]]], state=connected] 2024-09-14T06:57:53.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:53.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:53.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:53.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=14884], [object_number=47, tag_tid=8, entry_tid=14883], [object_number=46, tag_tid=8, entry_tid=14882], [object_number=45, tag_tid=8, entry_tid=14881]]], state=connected] 2024-09-14T06:57:53.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14884' '!=' 'object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=14884' = 'object_number=47, tag_tid=8, entry_tid=14783' 2024-09-14T06:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:53.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:53.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:53.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:53.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:53.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:53.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:53.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:53.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14884 2024-09-14T06:57:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14884 2024-09-14T06:57:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:54.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:54.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:54.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:54.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:54.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:54.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:54.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:54.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:54.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:54.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:54.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:54.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:54.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:54.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=14989], [object_number=44, tag_tid=8, entry_tid=14988], [object_number=47, tag_tid=8, entry_tid=14987], [object_number=46, tag_tid=8, entry_tid=14986]]], state=connected] 2024-09-14T06:57:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:54.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:54.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:54.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:54.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=14989], [object_number=44, tag_tid=8, entry_tid=14988], [object_number=47, tag_tid=8, entry_tid=14987], [object_number=46, tag_tid=8, entry_tid=14986]]], state=connected] 2024-09-14T06:57:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=14989' '!=' 'object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=14989' = 'object_number=44, tag_tid=8, entry_tid=14884' 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:54.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:54.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:54.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:54.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14989 2024-09-14T06:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 14989 2024-09-14T06:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:54.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:54.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:54.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=15090], [object_number=45, tag_tid=8, entry_tid=15089], [object_number=44, tag_tid=8, entry_tid=15088], [object_number=47, tag_tid=8, entry_tid=15087]]], state=connected] 2024-09-14T06:57:54.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:54.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:54.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:54.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:54.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:54.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:54.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:54.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=46, tag_tid=8, entry_tid=15090], [object_number=45, tag_tid=8, entry_tid=15089], [object_number=44, tag_tid=8, entry_tid=15088], [object_number=47, tag_tid=8, entry_tid=15087]]], state=connected] 2024-09-14T06:57:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:54.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=15090' '!=' 'object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=46, tag_tid=8, entry_tid=15090' = 'object_number=45, tag_tid=8, entry_tid=14989' 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:54.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:54.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:54.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15090 2024-09-14T06:57:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15090 2024-09-14T06:57:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:54.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:55.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:55.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:55.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:55.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:55.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:55.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:55.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:55.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:55.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:55.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:55.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:55.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:55.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:55.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:55.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:55.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:55.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:55.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:55.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=15193], [object_number=44, tag_tid=8, entry_tid=15192], [object_number=47, tag_tid=8, entry_tid=15191], [object_number=46, tag_tid=8, entry_tid=15190]]], state=connected] 2024-09-14T06:57:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:55.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:55.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:55.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=15193], [object_number=44, tag_tid=8, entry_tid=15192], [object_number=47, tag_tid=8, entry_tid=15191], [object_number=46, tag_tid=8, entry_tid=15190]]], state=connected] 2024-09-14T06:57:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=15193' '!=' 'object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:55.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=15193' = 'object_number=46, tag_tid=8, entry_tid=15090' 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:55.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:55.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:55.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15193 2024-09-14T06:57:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15193 2024-09-14T06:57:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:55.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:55.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:55.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:55.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:55.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:55.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:55.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:55.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:55.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:55.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:55.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:55.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=15305], [object_number=44, tag_tid=8, entry_tid=15304], [object_number=47, tag_tid=8, entry_tid=15303], [object_number=46, tag_tid=8, entry_tid=15302]]], state=connected] 2024-09-14T06:57:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=8, entry_tid=15305], [object_number=44, tag_tid=8, entry_tid=15304], [object_number=47, tag_tid=8, entry_tid=15303], [object_number=46, tag_tid=8, entry_tid=15302]]], state=connected] 2024-09-14T06:57:55.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=15305' '!=' 'object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=45, tag_tid=8, entry_tid=15305' = 'object_number=45, tag_tid=8, entry_tid=15193' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:55.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:55.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:55.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:55.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:55.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:55.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15305 2024-09-14T06:57:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15305 2024-09-14T06:57:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:56.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:56.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:56.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:56.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:56.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:56.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:56.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:56.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:56.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:56.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:56.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:56.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:56.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:56.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:56.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:56.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:56.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:56.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:56.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:56.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:56.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15411], [object_number=46, tag_tid=8, entry_tid=15410], [object_number=45, tag_tid=8, entry_tid=15409], [object_number=44, tag_tid=8, entry_tid=15408]]], state=connected] 2024-09-14T06:57:56.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:56.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:56.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:56.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15411], [object_number=46, tag_tid=8, entry_tid=15410], [object_number=45, tag_tid=8, entry_tid=15409], [object_number=44, tag_tid=8, entry_tid=15408]]], state=connected] 2024-09-14T06:57:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15411' '!=' 'object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15411' = 'object_number=45, tag_tid=8, entry_tid=15305' 2024-09-14T06:57:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:56.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:56.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:56.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:56.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:56.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:56.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:56.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:56.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15411 2024-09-14T06:57:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15411 2024-09-14T06:57:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:56.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:56.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:56.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:56.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:56.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:56.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:56.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:56.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:56.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:57.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:57.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:57.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:57.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=15564], [object_number=47, tag_tid=8, entry_tid=15563], [object_number=46, tag_tid=8, entry_tid=15562], [object_number=45, tag_tid=8, entry_tid=15561]]], state=connected] 2024-09-14T06:57:57.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:57.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=15564], [object_number=47, tag_tid=8, entry_tid=15563], [object_number=46, tag_tid=8, entry_tid=15562], [object_number=45, tag_tid=8, entry_tid=15561]]], state=connected] 2024-09-14T06:57:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=15564' '!=' 'object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=15564' = 'object_number=47, tag_tid=8, entry_tid=15411' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:57.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:57.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15564 2024-09-14T06:57:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15564 2024-09-14T06:57:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:57.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:57.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:57.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:57.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:57.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:57.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:57.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:57.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:57.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:57.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:57.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:57.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:57.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:57.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:57.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:57.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:57.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:57.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:57.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:57.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:57.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:57.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:57.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:57.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:57.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:57.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:57.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=15660], [object_number=47, tag_tid=8, entry_tid=15659], [object_number=46, tag_tid=8, entry_tid=15658], [object_number=45, tag_tid=8, entry_tid=15657]]], state=connected] 2024-09-14T06:57:57.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:57.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:57.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:57.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:57.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:57.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:57.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=44, tag_tid=8, entry_tid=15660], [object_number=47, tag_tid=8, entry_tid=15659], [object_number=46, tag_tid=8, entry_tid=15658], [object_number=45, tag_tid=8, entry_tid=15657]]], state=connected] 2024-09-14T06:57:57.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=15660' '!=' 'object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=44, tag_tid=8, entry_tid=15660' = 'object_number=44, tag_tid=8, entry_tid=15564' 2024-09-14T06:57:57.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:57.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:57.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:57.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:57.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:57.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:57.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:57.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:57.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:57.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:57.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15660 2024-09-14T06:57:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15660 2024-09-14T06:57:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:57.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:57.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:57.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:57.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:57.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:57.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:57.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:57.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:57.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:57.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:57.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:57.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:57.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15767], [object_number=46, tag_tid=8, entry_tid=15766], [object_number=45, tag_tid=8, entry_tid=15765], [object_number=44, tag_tid=8, entry_tid=15764]]], state=connected] 2024-09-14T06:57:58.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15767], [object_number=46, tag_tid=8, entry_tid=15766], [object_number=45, tag_tid=8, entry_tid=15765], [object_number=44, tag_tid=8, entry_tid=15764]]], state=connected] 2024-09-14T06:57:58.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15767' '!=' 'object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15767' = 'object_number=44, tag_tid=8, entry_tid=15660' 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:58.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:58.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:58.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:58.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15767 2024-09-14T06:57:58.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15767 2024-09-14T06:57:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:58.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:58.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:58.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:58.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:58.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:58.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:58.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:58.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890], [object_number=45, tag_tid=8, entry_tid=15889], [object_number=44, tag_tid=8, entry_tid=15888]]], state=connected] 2024-09-14T06:57:58.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:58.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:58.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890], [object_number=45, tag_tid=8, entry_tid=15889], [object_number=44, tag_tid=8, entry_tid=15888]]], state=connected] 2024-09-14T06:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:58.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15891' '!=' 'object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15891' = 'object_number=47, tag_tid=8, entry_tid=15767' 2024-09-14T06:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:58.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:58.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:58.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=9 2024-09-14T06:57:58.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=8 2024-09-14T06:57:58.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:58.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:58.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:57:58.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:58.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:57:58.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:57:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15891 2024-09-14T06:57:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 9 = 8 -a 1 -le 15891 2024-09-14T06:57:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:57:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:58.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:58.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:58.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:58.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:58.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:58.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:58.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:58.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:58.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:59.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:59.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890], [object_number=45, tag_tid=8, entry_tid=15889], [object_number=44, tag_tid=8, entry_tid=15888]]], state=connected] 2024-09-14T06:57:59.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:59.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:59.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:59.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:59.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890], [object_number=45, tag_tid=8, entry_tid=15889], [object_number=44, tag_tid=8, entry_tid=15888]]], state=connected] 2024-09-14T06:57:59.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:59.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:59.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=47, tag_tid=8, entry_tid=15891' '!=' 'object_number=47, tag_tid=8, entry_tid=15891' 2024-09-14T06:57:59.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:57:59.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T06:57:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:57:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:59.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:57:59.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:57:59.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:57:59.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:57:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:57:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:57:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:57:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:57:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:57:59.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:59.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:57:59.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:59.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:59.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:59.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:59.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:59.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:57:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:57:59.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:57:59.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:57:59.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 11 2024-09-14T06:57:59.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 11 2024-09-14T06:57:59.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:57:59.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:59.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=45, tag_tid=9, entry_tid=1], [object_number=44, tag_tid=9, entry_tid=0], [object_number=47, tag_tid=8, entry_tid=15891], [object_number=46, tag_tid=8, entry_tid=15890]]], state=connected] 2024-09-14T06:57:59.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=45, tag_tid=9, entry_tid=1' -a 'object_number=45, tag_tid=9, entry_tid=1' = 'object_number=45, tag_tid=9, entry_tid=1' 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T06:57:59.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:58:00.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2024-09-14T06:58:00.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 5 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 06:56:36 2024 2024-09-14T06:58:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 06:57:46 2024 2024-09-14T06:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T06:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T06:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T06:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T06:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:58:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap2-clean ']' 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap2-clean 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2-clean 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:01.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:01.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:01.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:02.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:02.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap3 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:03.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:03.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:04.479 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:04.475+0000 7fea4ab92640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:04.480 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:04.475+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T06:58:04.480 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:04.475+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:04.480 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:04.475+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:04.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:04.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:05.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap3-clean 2024-09-14T06:58:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap3-clean 2024-09-14T06:58:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3-clean 2024-09-14T06:58:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T06:58:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T06:58:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T06:58:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T06:58:06.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T06:58:06.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:58:06.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T06:58:08.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 41577472 bytes 2024-09-14T06:58:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 58303488 bytes 2024-09-14T06:58:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 73645056 bytes 2024-09-14T06:58:10.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 74267648 bytes 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:58:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:58:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:58:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:58:12.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T06:58:12.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:58:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T06:58:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T06:58:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:58:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T06:58:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":2963041.43,"entries_behind_primary":0,"entries_per_second":183.17,"non_primary_position":{"entry_tid":1,"object_number":45,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":9}} 2024-09-14T06:58:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T06:58:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:58:10 2024-09-14T06:58:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T06:58:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T06:58:12.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T06:58:12.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T06:58:12.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:58:10' 2024-09-14T06:58:12.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T06:58:12.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T06:58:12.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T06:58:12.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":2963041.43,"entries_behind_primary":0,"entries_per_second":183.17,"non_primary_position":{"entry_tid":1,"object_number":45,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":9}} 2024-09-14T06:58:12.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:58:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:58:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T06:58:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:58:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T06:58:12.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T06:58:12.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T06:58:12.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:58:10' 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 06:58:10' 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 06:58:10' 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:58:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T06:58:12.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK42184267mirror.2smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589trueOK43674370mirror.1smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":2963041.43,"entries_behind_primary":0,"entries_per_second":183.17,"non_primary_position":{"entry_tid":1,"object_number":45,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":9}}42104261mirror.0smithi1692024-09-14 06:58:10cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 06:58:10' 2024-09-14T06:58:12.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T06:58:12.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T06:58:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:58:10' 2024-09-14T06:58:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T06:58:12.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T06:58:12.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:58:12.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T06:58:12.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap4 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap4 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4 2024-09-14T06:58:12.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2024-09-14T06:58:42.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T06:58:42.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:42.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:58:43.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T06:58:43.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:43.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:43.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:43.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T06:58:43.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T06:58:43.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T06:58:43.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T06:58:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T06:58:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:58:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:58:43.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:58:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:43.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:43.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:43.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:43.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:58:43.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T06:58:43.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T06:58:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T06:58:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T06:58:43.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:58:43.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:58:43.715 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4267736990526493) > weight (0.25) 2024-09-14T06:58:43.715 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.320371499515278) > weight (0.25) 2024-09-14T06:58:43.715 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T06:58:43.716 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T06:58:43.716 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T06:58:43.717 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T06:58:43.715+0000 7f0199090640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 30431) UID: 1000 2024-09-14T06:58:43.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:58:43.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:43.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:43.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:43.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:58:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:58:43.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:58:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:43.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:43.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:43.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:43.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:58:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:58:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:43.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:58:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:58:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:43.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:43.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:43.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:43.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:43.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:43.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:43.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:43.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:43.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=53, tag_tid=12, entry_tid=1' -a 'object_number=53, tag_tid=12, entry_tid=1' = 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=54, tag_tid=11, entry_tid=6510' '!=' '' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=54, tag_tid=11, entry_tid=6510' = '' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:58:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=12 2024-09-14T06:58:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:58:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:58:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=11 2024-09-14T06:58:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:58:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:58:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:58:44.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:44.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:58:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:58:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=6510 2024-09-14T06:58:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 12 = 11 -a 1 -le 6510 2024-09-14T06:58:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:58:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:58:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:58:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:58:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:58:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:58:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:58:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:44.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:44.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:44.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:44.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:44.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:58:44.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:44.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:58:44.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:44.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:44.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:44.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:44.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:44.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:58:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:58:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:58:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=53, tag_tid=12, entry_tid=1' -a 'object_number=53, tag_tid=12, entry_tid=1' = 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=54, tag_tid=11, entry_tid=6510' '!=' 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:58:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T06:58:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:58:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:58:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:58:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:58:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:58:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:45.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:45.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:45.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:45.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:45.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:45.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:45.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:45.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:45.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:45.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:45.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=53, tag_tid=12, entry_tid=1' -a 'object_number=53, tag_tid=12, entry_tid=1' = 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:45.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=54, tag_tid=11, entry_tid=6510' '!=' 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:45.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:58:45.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T06:58:45.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:58:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:58:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:58:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:58:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:58:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:58:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:46.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:58:46.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:46.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:46.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:46.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:46.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:46.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:46.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:46.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:46.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:58:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:58:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:46.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=53, tag_tid=12, entry_tid=1' -a 'object_number=53, tag_tid=12, entry_tid=1' = 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=54, tag_tid=11, entry_tid=6510' '!=' 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:58:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T06:58:46.711 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.37596797783319524) > weight (0.25) 2024-09-14T06:58:46.711 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.413497279977712) > weight (0.25) 2024-09-14T06:58:46.711 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6900763841286953) > weight (0.25) 2024-09-14T06:58:46.712 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-09-14T06:58:46.712 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-09-14T06:58:46.712 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T06:58:46.713 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T06:58:46.711+0000 7f4ef24d0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 30564) UID: 1000 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:58:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:58:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:47.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:47.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:58:47.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:47.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:58:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:47.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:47.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:47.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:48.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:48.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:58:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:48.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:48.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:48.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:48.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:48.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=54, tag_tid=11, entry_tid=6510], [object_number=53, tag_tid=11, entry_tid=6509], [object_number=52, tag_tid=11, entry_tid=6508], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:48.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:48.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=53, tag_tid=12, entry_tid=1' -a 'object_number=53, tag_tid=12, entry_tid=1' = 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:48.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=54, tag_tid=11, entry_tid=6510' '!=' 'object_number=54, tag_tid=11, entry_tid=6510' 2024-09-14T06:58:48.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:58:48.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:58:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:58:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:58:50.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:58:50.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:58:50.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:58:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:58:50.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:58:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:58:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:58:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:58:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:58:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:58:50.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:50.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:50.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:58:50.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:58:50.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:50.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:50.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:58:50.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:50.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:50.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:50.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:50.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:50.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:50.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:50.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:50.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:58:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:58:50.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:50.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:58:50.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 13 2024-09-14T06:58:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 13 2024-09-14T06:58:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:58:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:50.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=53, tag_tid=12, entry_tid=1], [object_number=52, tag_tid=12, entry_tid=0], [object_number=54, tag_tid=11, entry_tid=6510], [object_number=55, tag_tid=11, entry_tid=6507]]], state=connected] 2024-09-14T06:58:50.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:58:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=53, tag_tid=12, entry_tid=1' -a 'object_number=53, tag_tid=12, entry_tid=1' = 'object_number=53, tag_tid=12, entry_tid=1' 2024-09-14T06:58:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T06:58:50.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:58:50.712 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-09-14T06:58:50.713 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:58:50.713 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-09-14T06:58:50.713 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-14T06:58:50.714 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-14T06:58:50.717 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-14T06:58:50.717 INFO:tasks.rbd_mirror_thrash:waiting for 71 secs before thrashing 2024-09-14T06:58:51.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 7 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T06:58:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 06:58:01 2024 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 06:57:46 2024 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:58:51.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T06:58:51.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:51.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap3-clean ']' 2024-09-14T06:58:51.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap3-clean 2024-09-14T06:58:51.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3-clean 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:51.717 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T06:58:51.717 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T06:58:51.718 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T06:58:51.718 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T06:58:51.718 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T06:58:51.720 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T06:58:51.720 INFO:tasks.rbd_mirror_thrash:waiting for 102 secs before thrashing 2024-09-14T06:58:52.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:52.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:52.407 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:52.402+0000 7fea5039d640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:52.407 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:52.402+0000 7fea4a391640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:52.409 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:52.406+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:53.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:53.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap4 2024-09-14T06:58:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T06:58:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:54.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:54.584 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:54.578+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T06:58:54.584 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:54.582+0000 7fea4db98640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2024-09-14T06:58:54.585 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:54.582+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:54.587 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:54.582+0000 7fea4b393640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T06:58:54.682 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T06:58:54.678+0000 7fea4b393640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T06:58:55.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T06:58:55.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:55.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T06:58:55.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap4-clean 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap4-clean 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4-clean 2024-09-14T06:58:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T06:58:56.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T06:58:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T06:58:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:58:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:58:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:58:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T06:58:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T06:58:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T06:58:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T06:58:58.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 49482752 bytes 2024-09-14T06:59:00.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 81261568 bytes 2024-09-14T06:59:01.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 104591360 bytes 2024-09-14T06:59:03.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 159049728 bytes 2024-09-14T06:59:04.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 201646080 bytes 2024-09-14T06:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:600 IOs, 203323392 bytes 2024-09-14T06:59:05.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:59:05.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:59:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:59:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:59:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T06:59:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:59:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T06:59:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T06:59:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:59:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:59:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T06:59:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":2472860.03,"entries_behind_primary":0,"entries_per_second":158.3,"non_primary_position":{"entry_tid":3999,"object_number":51,"tag_tid":11},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":10}} 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:58:40 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 06:58:40' 2024-09-14T06:59:06.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T06:59:06.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T06:59:06.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T06:59:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":2472860.03,"entries_behind_primary":0,"entries_per_second":158.3,"non_primary_position":{"entry_tid":3999,"object_number":51,"tag_tid":11},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":10}} 2024-09-14T06:59:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T06:59:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:59:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T06:59:06.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:59:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T06:59:06.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T06:59:06.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T06:59:06.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:58:40' 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 06:58:40' 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 06:58:40' 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T06:59:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T06:59:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589trueOK43674370mirror.1smithi16919.3.0-4775-g7f24e589falseOK44074410mirror.2smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":2472860.03,"entries_behind_primary":0,"entries_per_second":158.3,"non_primary_position":{"entry_tid":3999,"object_number":51,"tag_tid":11},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":10}}42104261mirror.0smithi1692024-09-14 06:58:40cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 06:58:40' 2024-09-14T06:59:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T06:59:06.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T06:59:06.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 06:58:40' 2024-09-14T06:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T06:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T06:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T06:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T06:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap5 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap5 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5 2024-09-14T06:59:06.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2024-09-14T06:59:09.398 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T06:59:09.394+0000 7f99ede58640 -1 reset not still connected to 0x5592105aec30 2024-09-14T06:59:09.401 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T06:59:09.398+0000 7f99ede58640 -1 reset not still connected to 0x5592115e49c0 2024-09-14T06:59:39.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T06:59:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T06:59:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:39.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T06:59:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T06:59:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T06:59:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T06:59:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T06:59:40.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T06:59:40.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T06:59:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T06:59:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T06:59:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:40.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:40.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:40.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:40.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:40.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:41.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:41.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=8468], [object_number=63, tag_tid=14, entry_tid=8467], [object_number=62, tag_tid=14, entry_tid=8466], [object_number=61, tag_tid=14, entry_tid=8465]]], state=connected] 2024-09-14T06:59:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:41.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:41.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:41.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=8468], [object_number=63, tag_tid=14, entry_tid=8467], [object_number=62, tag_tid=14, entry_tid=8466], [object_number=61, tag_tid=14, entry_tid=8465]]], state=connected] 2024-09-14T06:59:41.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=8468' '!=' '' 2024-09-14T06:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=8468' = '' 2024-09-14T06:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8468 2024-09-14T06:59:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 8468 2024-09-14T06:59:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:41.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:41.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:41.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:41.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:41.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:41.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:41.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:41.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:41.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:41.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:41.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:41.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:41.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:41.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:41.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:41.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:41.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:41.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=8581], [object_number=60, tag_tid=14, entry_tid=8580], [object_number=63, tag_tid=14, entry_tid=8579], [object_number=62, tag_tid=14, entry_tid=8578]]], state=connected] 2024-09-14T06:59:41.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:41.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:41.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:41.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:41.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=8581], [object_number=60, tag_tid=14, entry_tid=8580], [object_number=63, tag_tid=14, entry_tid=8579], [object_number=62, tag_tid=14, entry_tid=8578]]], state=connected] 2024-09-14T06:59:41.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=8581' '!=' 'object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=8581' = 'object_number=60, tag_tid=14, entry_tid=8468' 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:41.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:41.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:41.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:41.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:41.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:41.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8581 2024-09-14T06:59:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 8581 2024-09-14T06:59:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:41.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:41.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:41.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:42.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:42.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:42.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:42.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:42.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=8698], [object_number=61, tag_tid=14, entry_tid=8697], [object_number=60, tag_tid=14, entry_tid=8696], [object_number=63, tag_tid=14, entry_tid=8695]]], state=connected] 2024-09-14T06:59:42.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:42.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:42.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:42.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:42.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:42.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:42.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:42.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=8698], [object_number=61, tag_tid=14, entry_tid=8697], [object_number=60, tag_tid=14, entry_tid=8696], [object_number=63, tag_tid=14, entry_tid=8695]]], state=connected] 2024-09-14T06:59:42.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=8698' '!=' 'object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=8698' = 'object_number=61, tag_tid=14, entry_tid=8581' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:42.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:42.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8698 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 8698 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:42.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:42.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:42.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:42.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:42.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:42.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:42.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:42.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:42.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:42.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:42.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:42.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:42.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:42.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=8800], [object_number=63, tag_tid=14, entry_tid=8799], [object_number=62, tag_tid=14, entry_tid=8798], [object_number=61, tag_tid=14, entry_tid=8797]]], state=connected] 2024-09-14T06:59:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:42.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=8800], [object_number=63, tag_tid=14, entry_tid=8799], [object_number=62, tag_tid=14, entry_tid=8798], [object_number=61, tag_tid=14, entry_tid=8797]]], state=connected] 2024-09-14T06:59:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=8800' '!=' 'object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=8800' = 'object_number=62, tag_tid=14, entry_tid=8698' 2024-09-14T06:59:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:42.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:42.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:42.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:42.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:42.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:42.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:42.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:42.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:42.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8800 2024-09-14T06:59:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 8800 2024-09-14T06:59:42.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:42.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:42.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:42.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:42.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:42.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:42.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:42.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:42.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:42.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:42.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=8895], [object_number=62, tag_tid=14, entry_tid=8894], [object_number=61, tag_tid=14, entry_tid=8893], [object_number=60, tag_tid=14, entry_tid=8892]]], state=connected] 2024-09-14T06:59:43.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:43.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=8895], [object_number=62, tag_tid=14, entry_tid=8894], [object_number=61, tag_tid=14, entry_tid=8893], [object_number=60, tag_tid=14, entry_tid=8892]]], state=connected] 2024-09-14T06:59:43.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=8895' '!=' 'object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=8895' = 'object_number=60, tag_tid=14, entry_tid=8800' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:43.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:43.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:43.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8895 2024-09-14T06:59:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 8895 2024-09-14T06:59:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:43.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:43.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:43.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:43.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:43.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:43.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:43.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:43.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:43.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:43.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:43.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:43.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:43.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:43.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:43.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9000], [object_number=63, tag_tid=14, entry_tid=8999], [object_number=62, tag_tid=14, entry_tid=8998], [object_number=61, tag_tid=14, entry_tid=8997]]], state=connected] 2024-09-14T06:59:43.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:43.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9000], [object_number=63, tag_tid=14, entry_tid=8999], [object_number=62, tag_tid=14, entry_tid=8998], [object_number=61, tag_tid=14, entry_tid=8997]]], state=connected] 2024-09-14T06:59:43.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:43.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:43.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:43.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9000' '!=' 'object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9000' = 'object_number=63, tag_tid=14, entry_tid=8895' 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:43.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:43.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:43.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9000 2024-09-14T06:59:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9000 2024-09-14T06:59:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:43.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:43.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:43.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:43.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:43.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:43.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:43.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:44.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:44.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:44.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:44.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:44.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:44.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9108], [object_number=63, tag_tid=14, entry_tid=9107], [object_number=62, tag_tid=14, entry_tid=9106], [object_number=61, tag_tid=14, entry_tid=9105]]], state=connected] 2024-09-14T06:59:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:44.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:44.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9108], [object_number=63, tag_tid=14, entry_tid=9107], [object_number=62, tag_tid=14, entry_tid=9106], [object_number=61, tag_tid=14, entry_tid=9105]]], state=connected] 2024-09-14T06:59:44.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9108' '!=' 'object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:44.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9108' = 'object_number=60, tag_tid=14, entry_tid=9000' 2024-09-14T06:59:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:44.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:44.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9108 2024-09-14T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9108 2024-09-14T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:44.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:44.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:44.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:44.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:44.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:44.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:44.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:44.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:44.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:44.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:44.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:44.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:44.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:44.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:44.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:44.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=9225], [object_number=60, tag_tid=14, entry_tid=9224], [object_number=63, tag_tid=14, entry_tid=9223], [object_number=62, tag_tid=14, entry_tid=9222]]], state=connected] 2024-09-14T06:59:44.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=9225], [object_number=60, tag_tid=14, entry_tid=9224], [object_number=63, tag_tid=14, entry_tid=9223], [object_number=62, tag_tid=14, entry_tid=9222]]], state=connected] 2024-09-14T06:59:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=9225' '!=' 'object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=9225' = 'object_number=60, tag_tid=14, entry_tid=9108' 2024-09-14T06:59:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:44.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:44.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:44.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:44.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9225 2024-09-14T06:59:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9225 2024-09-14T06:59:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:44.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:44.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:44.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:44.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:44.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:44.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:44.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:44.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:44.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:44.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:44.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:44.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:44.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:45.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:45.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=9335], [object_number=62, tag_tid=14, entry_tid=9334], [object_number=61, tag_tid=14, entry_tid=9333], [object_number=60, tag_tid=14, entry_tid=9332]]], state=connected] 2024-09-14T06:59:45.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:45.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=9335], [object_number=62, tag_tid=14, entry_tid=9334], [object_number=61, tag_tid=14, entry_tid=9333], [object_number=60, tag_tid=14, entry_tid=9332]]], state=connected] 2024-09-14T06:59:45.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=9335' '!=' 'object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=9335' = 'object_number=61, tag_tid=14, entry_tid=9225' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:45.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:45.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:45.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:45.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:45.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9335 2024-09-14T06:59:45.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9335 2024-09-14T06:59:45.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:45.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:45.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:45.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:45.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:45.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:45.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:45.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:45.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=9450], [object_number=61, tag_tid=14, entry_tid=9449], [object_number=60, tag_tid=14, entry_tid=9448], [object_number=63, tag_tid=14, entry_tid=9447]]], state=connected] 2024-09-14T06:59:45.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:45.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=9450], [object_number=61, tag_tid=14, entry_tid=9449], [object_number=60, tag_tid=14, entry_tid=9448], [object_number=63, tag_tid=14, entry_tid=9447]]], state=connected] 2024-09-14T06:59:45.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=9450' '!=' 'object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=9450' = 'object_number=63, tag_tid=14, entry_tid=9335' 2024-09-14T06:59:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:45.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:45.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:45.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:45.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:45.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:45.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:45.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:45.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:45.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:45.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:45.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:45.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9450 2024-09-14T06:59:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9450 2024-09-14T06:59:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:45.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:45.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9560], [object_number=63, tag_tid=14, entry_tid=9559], [object_number=62, tag_tid=14, entry_tid=9558], [object_number=61, tag_tid=14, entry_tid=9557]]], state=connected] 2024-09-14T06:59:46.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9560], [object_number=63, tag_tid=14, entry_tid=9559], [object_number=62, tag_tid=14, entry_tid=9558], [object_number=61, tag_tid=14, entry_tid=9557]]], state=connected] 2024-09-14T06:59:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9560' '!=' 'object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9560' = 'object_number=62, tag_tid=14, entry_tid=9450' 2024-09-14T06:59:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:46.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:46.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9560 2024-09-14T06:59:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9560 2024-09-14T06:59:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:46.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:46.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:46.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:46.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:46.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:46.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:46.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:46.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:46.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:46.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:46.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:46.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:46.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:46.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:46.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:46.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:46.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:46.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:46.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:46.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:46.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:46.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:46.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:46.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:46.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:46.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:46.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:46.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:46.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:46.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:46.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9668], [object_number=63, tag_tid=14, entry_tid=9667], [object_number=62, tag_tid=14, entry_tid=9666], [object_number=61, tag_tid=14, entry_tid=9665]]], state=connected] 2024-09-14T06:59:46.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:46.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:46.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9668], [object_number=63, tag_tid=14, entry_tid=9667], [object_number=62, tag_tid=14, entry_tid=9666], [object_number=61, tag_tid=14, entry_tid=9665]]], state=connected] 2024-09-14T06:59:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:46.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:46.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9668' '!=' 'object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9668' = 'object_number=60, tag_tid=14, entry_tid=9560' 2024-09-14T06:59:46.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:46.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:46.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:46.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:46.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:46.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:46.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:46.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:46.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:46.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:46.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9668 2024-09-14T06:59:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9668 2024-09-14T06:59:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:46.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:46.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:46.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:46.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:46.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:46.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:46.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:46.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:46.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:46.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:46.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:46.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:46.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:46.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:46.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:46.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9772], [object_number=63, tag_tid=14, entry_tid=9771], [object_number=62, tag_tid=14, entry_tid=9770], [object_number=61, tag_tid=14, entry_tid=9769]]], state=connected] 2024-09-14T06:59:46.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:46.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:46.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:46.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:46.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:46.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:47.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:47.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:47.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:47.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9772], [object_number=63, tag_tid=14, entry_tid=9771], [object_number=62, tag_tid=14, entry_tid=9770], [object_number=61, tag_tid=14, entry_tid=9769]]], state=connected] 2024-09-14T06:59:47.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9772' '!=' 'object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:47.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9772' = 'object_number=60, tag_tid=14, entry_tid=9668' 2024-09-14T06:59:47.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:47.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:47.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:47.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:47.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9772 2024-09-14T06:59:47.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9772 2024-09-14T06:59:47.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:47.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:47.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:47.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:47.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:47.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:47.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:47.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:47.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:47.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:47.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:47.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:47.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:47.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:47.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=9877], [object_number=60, tag_tid=14, entry_tid=9876], [object_number=63, tag_tid=14, entry_tid=9875], [object_number=62, tag_tid=14, entry_tid=9874]]], state=connected] 2024-09-14T06:59:47.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:47.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:47.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:47.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:47.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=9877], [object_number=60, tag_tid=14, entry_tid=9876], [object_number=63, tag_tid=14, entry_tid=9875], [object_number=62, tag_tid=14, entry_tid=9874]]], state=connected] 2024-09-14T06:59:47.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:47.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=9877' '!=' 'object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:47.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=9877' = 'object_number=60, tag_tid=14, entry_tid=9772' 2024-09-14T06:59:47.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:47.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:47.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:47.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:47.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:47.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:47.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:47.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:47.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:47.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9877 2024-09-14T06:59:47.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9877 2024-09-14T06:59:47.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:47.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:47.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:47.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:47.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:47.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:47.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:47.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:47.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:47.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:47.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:47.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:47.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9984], [object_number=63, tag_tid=14, entry_tid=9983], [object_number=62, tag_tid=14, entry_tid=9982], [object_number=61, tag_tid=14, entry_tid=9981]]], state=connected] 2024-09-14T06:59:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:47.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:47.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:47.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=9984], [object_number=63, tag_tid=14, entry_tid=9983], [object_number=62, tag_tid=14, entry_tid=9982], [object_number=61, tag_tid=14, entry_tid=9981]]], state=connected] 2024-09-14T06:59:48.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9984' '!=' 'object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=9984' = 'object_number=61, tag_tid=14, entry_tid=9877' 2024-09-14T06:59:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:48.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:48.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9984 2024-09-14T06:59:48.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 9984 2024-09-14T06:59:48.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:48.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:48.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:48.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:48.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:48.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:48.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:48.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:48.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:48.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:48.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:48.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:48.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:48.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:48.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:48.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:48.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:48.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:48.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:48.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:48.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:48.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:48.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=10077], [object_number=60, tag_tid=14, entry_tid=10076], [object_number=63, tag_tid=14, entry_tid=10075], [object_number=62, tag_tid=14, entry_tid=10074]]], state=connected] 2024-09-14T06:59:48.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:48.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:48.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:48.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=10077], [object_number=60, tag_tid=14, entry_tid=10076], [object_number=63, tag_tid=14, entry_tid=10075], [object_number=62, tag_tid=14, entry_tid=10074]]], state=connected] 2024-09-14T06:59:48.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=10077' '!=' 'object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=10077' = 'object_number=60, tag_tid=14, entry_tid=9984' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:48.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:48.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:48.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:48.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:48.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:48.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:48.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:48.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:48.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10077 2024-09-14T06:59:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10077 2024-09-14T06:59:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:48.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:48.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:48.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:48.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:48.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:48.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:48.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:48.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:48.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:48.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:48.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:48.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=10162], [object_number=61, tag_tid=14, entry_tid=10161], [object_number=60, tag_tid=14, entry_tid=10160], [object_number=63, tag_tid=14, entry_tid=10159]]], state=connected] 2024-09-14T06:59:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:48.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:48.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:48.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:48.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=10162], [object_number=61, tag_tid=14, entry_tid=10161], [object_number=60, tag_tid=14, entry_tid=10160], [object_number=63, tag_tid=14, entry_tid=10159]]], state=connected] 2024-09-14T06:59:49.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=10162' '!=' 'object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:49.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:49.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=10162' = 'object_number=61, tag_tid=14, entry_tid=10077' 2024-09-14T06:59:49.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:49.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:49.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10162 2024-09-14T06:59:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10162 2024-09-14T06:59:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:49.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:49.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:49.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:49.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:49.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:49.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:49.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:49.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:49.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=10248], [object_number=63, tag_tid=14, entry_tid=10247], [object_number=62, tag_tid=14, entry_tid=10246], [object_number=61, tag_tid=14, entry_tid=10245]]], state=connected] 2024-09-14T06:59:49.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:49.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:49.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:49.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:49.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=10248], [object_number=63, tag_tid=14, entry_tid=10247], [object_number=62, tag_tid=14, entry_tid=10246], [object_number=61, tag_tid=14, entry_tid=10245]]], state=connected] 2024-09-14T06:59:49.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=10248' '!=' 'object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:49.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=10248' = 'object_number=62, tag_tid=14, entry_tid=10162' 2024-09-14T06:59:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:49.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:49.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:49.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:49.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:49.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:49.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10248 2024-09-14T06:59:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10248 2024-09-14T06:59:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:49.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:49.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:49.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:49.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:49.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:49.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:49.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:49.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:49.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:49.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:49.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:49.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:49.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=10339], [object_number=62, tag_tid=14, entry_tid=10338], [object_number=61, tag_tid=14, entry_tid=10337], [object_number=60, tag_tid=14, entry_tid=10336]]], state=connected] 2024-09-14T06:59:49.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:49.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:49.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:49.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:49.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=10339], [object_number=62, tag_tid=14, entry_tid=10338], [object_number=61, tag_tid=14, entry_tid=10337], [object_number=60, tag_tid=14, entry_tid=10336]]], state=connected] 2024-09-14T06:59:49.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:49.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:49.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:49.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=10339' '!=' 'object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:49.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=10339' = 'object_number=60, tag_tid=14, entry_tid=10248' 2024-09-14T06:59:49.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:49.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:49.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:49.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:49.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:49.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:49.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:49.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:49.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:49.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:49.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:49.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:49.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10339 2024-09-14T06:59:49.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10339 2024-09-14T06:59:49.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:50.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:50.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:50.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:50.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:50.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:50.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:50.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:50.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:50.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:50.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=10452], [object_number=63, tag_tid=14, entry_tid=10451], [object_number=62, tag_tid=14, entry_tid=10450], [object_number=61, tag_tid=14, entry_tid=10449]]], state=connected] 2024-09-14T06:59:50.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:50.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:50.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:50.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:50.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:50.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=10452], [object_number=63, tag_tid=14, entry_tid=10451], [object_number=62, tag_tid=14, entry_tid=10450], [object_number=61, tag_tid=14, entry_tid=10449]]], state=connected] 2024-09-14T06:59:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=10452' '!=' 'object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:50.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:50.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=10452' = 'object_number=63, tag_tid=14, entry_tid=10339' 2024-09-14T06:59:50.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:50.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:50.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:50.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:50.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:50.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:50.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:50.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10452 2024-09-14T06:59:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10452 2024-09-14T06:59:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:50.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:50.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:50.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:50.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:50.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:50.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:50.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:50.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:50.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:50.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:50.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:50.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:50.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:50.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:50.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=10574], [object_number=61, tag_tid=14, entry_tid=10573], [object_number=60, tag_tid=14, entry_tid=10572], [object_number=63, tag_tid=14, entry_tid=10571]]], state=connected] 2024-09-14T06:59:50.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:50.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:50.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:50.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:50.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:50.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:50.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:50.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:50.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:50.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:50.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=62, tag_tid=14, entry_tid=10574], [object_number=61, tag_tid=14, entry_tid=10573], [object_number=60, tag_tid=14, entry_tid=10572], [object_number=63, tag_tid=14, entry_tid=10571]]], state=connected] 2024-09-14T06:59:50.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:50.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:50.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:50.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=10574' '!=' 'object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:50.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=62, tag_tid=14, entry_tid=10574' = 'object_number=60, tag_tid=14, entry_tid=10452' 2024-09-14T06:59:50.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:50.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:50.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:50.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:50.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:50.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:50.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:50.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:50.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:50.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:50.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:50.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:50.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10574 2024-09-14T06:59:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10574 2024-09-14T06:59:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:50.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:51.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:51.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:51.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=10677], [object_number=60, tag_tid=14, entry_tid=10676], [object_number=63, tag_tid=14, entry_tid=10675], [object_number=62, tag_tid=14, entry_tid=10674]]], state=connected] 2024-09-14T06:59:51.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:51.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:51.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:51.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=10677], [object_number=60, tag_tid=14, entry_tid=10676], [object_number=63, tag_tid=14, entry_tid=10675], [object_number=62, tag_tid=14, entry_tid=10674]]], state=connected] 2024-09-14T06:59:51.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=10677' '!=' 'object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:51.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:51.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=10677' = 'object_number=62, tag_tid=14, entry_tid=10574' 2024-09-14T06:59:51.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:51.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:51.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:51.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:51.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:51.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:51.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10677 2024-09-14T06:59:51.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10677 2024-09-14T06:59:51.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:51.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:51.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:51.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:51.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:51.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:51.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:51.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:51.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:51.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=10771], [object_number=62, tag_tid=14, entry_tid=10770], [object_number=61, tag_tid=14, entry_tid=10769], [object_number=60, tag_tid=14, entry_tid=10768]]], state=connected] 2024-09-14T06:59:51.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:51.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:51.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=10771], [object_number=62, tag_tid=14, entry_tid=10770], [object_number=61, tag_tid=14, entry_tid=10769], [object_number=60, tag_tid=14, entry_tid=10768]]], state=connected] 2024-09-14T06:59:51.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=10771' '!=' 'object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=10771' = 'object_number=61, tag_tid=14, entry_tid=10677' 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:51.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:51.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:51.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:51.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:51.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10771 2024-09-14T06:59:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10771 2024-09-14T06:59:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:52.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:52.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:52.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:52.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:52.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:52.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:52.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:52.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:52.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:52.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:52.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:52.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=10876], [object_number=63, tag_tid=14, entry_tid=10875], [object_number=62, tag_tid=14, entry_tid=10874], [object_number=61, tag_tid=14, entry_tid=10873]]], state=connected] 2024-09-14T06:59:52.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=10876], [object_number=63, tag_tid=14, entry_tid=10875], [object_number=62, tag_tid=14, entry_tid=10874], [object_number=61, tag_tid=14, entry_tid=10873]]], state=connected] 2024-09-14T06:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=10876' '!=' 'object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:52.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=10876' = 'object_number=63, tag_tid=14, entry_tid=10771' 2024-09-14T06:59:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:52.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10876 2024-09-14T06:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10876 2024-09-14T06:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:52.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:52.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:52.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:52.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:52.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:52.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:52.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:52.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:52.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:52.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:52.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:52.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:52.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:52.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:52.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:52.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:52.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:52.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:52.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:52.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:52.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:52.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:52.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=10979], [object_number=62, tag_tid=14, entry_tid=10978], [object_number=61, tag_tid=14, entry_tid=10977], [object_number=60, tag_tid=14, entry_tid=10976]]], state=connected] 2024-09-14T06:59:52.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:52.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:52.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:52.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:52.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:52.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:52.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:52.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:52.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:52.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=10979], [object_number=62, tag_tid=14, entry_tid=10978], [object_number=61, tag_tid=14, entry_tid=10977], [object_number=60, tag_tid=14, entry_tid=10976]]], state=connected] 2024-09-14T06:59:52.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=10979' '!=' 'object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=10979' = 'object_number=60, tag_tid=14, entry_tid=10876' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:52.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:52.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:52.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:52.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:52.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:52.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:52.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10979 2024-09-14T06:59:52.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 10979 2024-09-14T06:59:52.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:52.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:52.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:53.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:53.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:53.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:53.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:53.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:53.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:53.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:53.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:53.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:53.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:53.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:53.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:53.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:53.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:53.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:53.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:53.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:53.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=11083], [object_number=62, tag_tid=14, entry_tid=11082], [object_number=61, tag_tid=14, entry_tid=11081], [object_number=60, tag_tid=14, entry_tid=11080]]], state=connected] 2024-09-14T06:59:53.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:53.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:53.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:53.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:53.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:53.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:53.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:53.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=63, tag_tid=14, entry_tid=11083], [object_number=62, tag_tid=14, entry_tid=11082], [object_number=61, tag_tid=14, entry_tid=11081], [object_number=60, tag_tid=14, entry_tid=11080]]], state=connected] 2024-09-14T06:59:53.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=11083' '!=' 'object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=63, tag_tid=14, entry_tid=11083' = 'object_number=63, tag_tid=14, entry_tid=10979' 2024-09-14T06:59:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:53.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:53.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:53.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:53.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11083 2024-09-14T06:59:53.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11083 2024-09-14T06:59:53.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:53.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:53.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:53.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:53.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:53.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:53.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:53.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:53.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:53.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:53.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:53.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=11181], [object_number=60, tag_tid=14, entry_tid=11180], [object_number=63, tag_tid=14, entry_tid=11179], [object_number=62, tag_tid=14, entry_tid=11178]]], state=connected] 2024-09-14T06:59:53.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:53.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:53.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:53.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=61, tag_tid=14, entry_tid=11181], [object_number=60, tag_tid=14, entry_tid=11180], [object_number=63, tag_tid=14, entry_tid=11179], [object_number=62, tag_tid=14, entry_tid=11178]]], state=connected] 2024-09-14T06:59:53.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:53.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:53.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:53.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=11181' '!=' 'object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:53.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=61, tag_tid=14, entry_tid=11181' = 'object_number=63, tag_tid=14, entry_tid=11083' 2024-09-14T06:59:53.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:53.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:53.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:53.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:53.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:53.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:53.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11181 2024-09-14T06:59:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11181 2024-09-14T06:59:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:54.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:54.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:54.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:54.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:54.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:54.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:54.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:54.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:54.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:54.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:54.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:54.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=11276], [object_number=63, tag_tid=14, entry_tid=11275], [object_number=62, tag_tid=14, entry_tid=11274], [object_number=61, tag_tid=14, entry_tid=11273]]], state=connected] 2024-09-14T06:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:54.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:54.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:54.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:54.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:54.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=11276], [object_number=63, tag_tid=14, entry_tid=11275], [object_number=62, tag_tid=14, entry_tid=11274], [object_number=61, tag_tid=14, entry_tid=11273]]], state=connected] 2024-09-14T06:59:54.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=11276' '!=' 'object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=11276' = 'object_number=61, tag_tid=14, entry_tid=11181' 2024-09-14T06:59:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:54.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:54.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:54.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:54.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:54.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:54.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:54.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11276 2024-09-14T06:59:54.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11276 2024-09-14T06:59:54.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:54.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:54.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:54.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:54.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:54.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:54.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:54.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:54.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:54.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:54.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:54.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:54.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:54.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:54.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:54.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=11376], [object_number=63, tag_tid=14, entry_tid=11375], [object_number=62, tag_tid=14, entry_tid=11374], [object_number=61, tag_tid=14, entry_tid=11373]]], state=connected] 2024-09-14T06:59:54.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:54.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:54.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=11376], [object_number=63, tag_tid=14, entry_tid=11375], [object_number=62, tag_tid=14, entry_tid=11374], [object_number=61, tag_tid=14, entry_tid=11373]]], state=connected] 2024-09-14T06:59:54.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:54.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:54.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:54.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=11376' '!=' 'object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:54.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=11376' = 'object_number=60, tag_tid=14, entry_tid=11276' 2024-09-14T06:59:54.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:54.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:54.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:54.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:54.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:54.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:54.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:54.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:54.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11376 2024-09-14T06:59:54.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11376 2024-09-14T06:59:54.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:54.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:54.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:55.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:55.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:55.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=11480], [object_number=63, tag_tid=14, entry_tid=11479], [object_number=62, tag_tid=14, entry_tid=11478], [object_number=61, tag_tid=14, entry_tid=11477]]], state=connected] 2024-09-14T06:59:55.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:55.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:55.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 15 2024-09-14T06:59:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=60, tag_tid=14, entry_tid=11480], [object_number=63, tag_tid=14, entry_tid=11479], [object_number=62, tag_tid=14, entry_tid=11478], [object_number=61, tag_tid=14, entry_tid=11477]]], state=connected] 2024-09-14T06:59:55.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=11480' '!=' 'object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=60, tag_tid=14, entry_tid=11480' = 'object_number=60, tag_tid=14, entry_tid=11376' 2024-09-14T06:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:55.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:55.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:55.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:55.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:55.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11480 2024-09-14T06:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11480 2024-09-14T06:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:55.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:55.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:55.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:55.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:55.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:55.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:55.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:55.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:55.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:55.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:55.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:55.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:55.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=11558], [object_number=65, tag_tid=14, entry_tid=11557], [object_number=64, tag_tid=14, entry_tid=11556], [object_number=67, tag_tid=14, entry_tid=11555]]], state=connected] 2024-09-14T06:59:55.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:55.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=11558], [object_number=65, tag_tid=14, entry_tid=11557], [object_number=64, tag_tid=14, entry_tid=11556], [object_number=67, tag_tid=14, entry_tid=11555]]], state=connected] 2024-09-14T06:59:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=11558' '!=' 'object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=11558' = 'object_number=60, tag_tid=14, entry_tid=11480' 2024-09-14T06:59:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11558 2024-09-14T06:59:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11558 2024-09-14T06:59:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:56.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:56.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:56.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=11660], [object_number=67, tag_tid=14, entry_tid=11659], [object_number=66, tag_tid=14, entry_tid=11658], [object_number=65, tag_tid=14, entry_tid=11657]]], state=connected] 2024-09-14T06:59:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:56.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:56.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:56.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:56.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=11660], [object_number=67, tag_tid=14, entry_tid=11659], [object_number=66, tag_tid=14, entry_tid=11658], [object_number=65, tag_tid=14, entry_tid=11657]]], state=connected] 2024-09-14T06:59:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=11660' '!=' 'object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=11660' = 'object_number=66, tag_tid=14, entry_tid=11558' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11660 2024-09-14T06:59:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11660 2024-09-14T06:59:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:56.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:56.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:56.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:56.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:56.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:56.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:56.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:56.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:56.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:56.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:56.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:56.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:56.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:56.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:56.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:56.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:56.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:56.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=11763], [object_number=66, tag_tid=14, entry_tid=11762], [object_number=65, tag_tid=14, entry_tid=11761], [object_number=64, tag_tid=14, entry_tid=11760]]], state=connected] 2024-09-14T06:59:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:56.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:56.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:56.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:56.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=11763], [object_number=66, tag_tid=14, entry_tid=11762], [object_number=65, tag_tid=14, entry_tid=11761], [object_number=64, tag_tid=14, entry_tid=11760]]], state=connected] 2024-09-14T06:59:56.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:56.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:56.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:56.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=11763' '!=' 'object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:56.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=11763' = 'object_number=64, tag_tid=14, entry_tid=11660' 2024-09-14T06:59:56.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:56.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:56.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:56.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:56.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:56.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:56.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11763 2024-09-14T06:59:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11763 2024-09-14T06:59:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:56.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:56.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:57.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:57.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:57.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=11861], [object_number=64, tag_tid=14, entry_tid=11860], [object_number=67, tag_tid=14, entry_tid=11859], [object_number=66, tag_tid=14, entry_tid=11858]]], state=connected] 2024-09-14T06:59:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:57.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=11861], [object_number=64, tag_tid=14, entry_tid=11860], [object_number=67, tag_tid=14, entry_tid=11859], [object_number=66, tag_tid=14, entry_tid=11858]]], state=connected] 2024-09-14T06:59:57.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=11861' '!=' 'object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=11861' = 'object_number=67, tag_tid=14, entry_tid=11763' 2024-09-14T06:59:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:57.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:57.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:57.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:57.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11861 2024-09-14T06:59:57.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11861 2024-09-14T06:59:57.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:57.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:57.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:57.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:57.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:57.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:57.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:57.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:57.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:57.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:57.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:57.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:57.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=11957], [object_number=64, tag_tid=14, entry_tid=11956], [object_number=67, tag_tid=14, entry_tid=11955], [object_number=66, tag_tid=14, entry_tid=11954]]], state=connected] 2024-09-14T06:59:57.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:57.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:57.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:57.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:57.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:57.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:57.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:57.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:57.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:57.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:57.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:57.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:57.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=11957], [object_number=64, tag_tid=14, entry_tid=11956], [object_number=67, tag_tid=14, entry_tid=11955], [object_number=66, tag_tid=14, entry_tid=11954]]], state=connected] 2024-09-14T06:59:57.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:57.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:57.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:57.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=11957' '!=' 'object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=11957' = 'object_number=65, tag_tid=14, entry_tid=11861' 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:57.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:57.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:57.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11957 2024-09-14T06:59:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 11957 2024-09-14T06:59:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:57.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:57.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:57.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:57.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:57.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:57.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:57.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:57.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:57.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:57.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:57.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:57.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:57.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:57.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:57.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:57.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:57.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:57.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:57.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:58.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:58.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:58.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:58.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=12056], [object_number=67, tag_tid=14, entry_tid=12055], [object_number=66, tag_tid=14, entry_tid=12054], [object_number=65, tag_tid=14, entry_tid=12053]]], state=connected] 2024-09-14T06:59:58.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:58.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:58.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:58.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:58.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:58.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:58.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:58.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:58.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:58.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=12056], [object_number=67, tag_tid=14, entry_tid=12055], [object_number=66, tag_tid=14, entry_tid=12054], [object_number=65, tag_tid=14, entry_tid=12053]]], state=connected] 2024-09-14T06:59:58.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=12056' '!=' 'object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:58.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:58.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=12056' = 'object_number=65, tag_tid=14, entry_tid=11957' 2024-09-14T06:59:58.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:58.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:58.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:58.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:58.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:58.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:58.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:58.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:58.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:58.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12056 2024-09-14T06:59:58.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12056 2024-09-14T06:59:58.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:58.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:58.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:58.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:58.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:58.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:58.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:58.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:58.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:58.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:58.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:58.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:58.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:58.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:58.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:58.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:58.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:58.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:58.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:58.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:58.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12166], [object_number=65, tag_tid=14, entry_tid=12165], [object_number=64, tag_tid=14, entry_tid=12164], [object_number=67, tag_tid=14, entry_tid=12163]]], state=connected] 2024-09-14T06:59:58.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:58.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12166], [object_number=65, tag_tid=14, entry_tid=12165], [object_number=64, tag_tid=14, entry_tid=12164], [object_number=67, tag_tid=14, entry_tid=12163]]], state=connected] 2024-09-14T06:59:58.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:58.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:58.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:58.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12166' '!=' 'object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12166' = 'object_number=64, tag_tid=14, entry_tid=12056' 2024-09-14T06:59:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:58.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:58.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:58.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12166 2024-09-14T06:59:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12166 2024-09-14T06:59:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:58.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:58.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:58.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:58.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:58.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:58.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:58.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:58.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:58.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:58.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:58.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:59.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:59.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:59.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:59.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:59.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:59.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:59.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:59.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12274], [object_number=65, tag_tid=14, entry_tid=12273], [object_number=64, tag_tid=14, entry_tid=12272], [object_number=67, tag_tid=14, entry_tid=12271]]], state=connected] 2024-09-14T06:59:59.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:59.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:59.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:59.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:59.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12274], [object_number=65, tag_tid=14, entry_tid=12273], [object_number=64, tag_tid=14, entry_tid=12272], [object_number=67, tag_tid=14, entry_tid=12271]]], state=connected] 2024-09-14T06:59:59.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12274' '!=' 'object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12274' = 'object_number=66, tag_tid=14, entry_tid=12166' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:59.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:59.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:59.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12274 2024-09-14T06:59:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12274 2024-09-14T06:59:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:59.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:59.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T06:59:59.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T06:59:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T06:59:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=12379], [object_number=66, tag_tid=14, entry_tid=12378], [object_number=65, tag_tid=14, entry_tid=12377], [object_number=64, tag_tid=14, entry_tid=12376]]], state=connected] 2024-09-14T06:59:59.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:59.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T06:59:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T06:59:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T06:59:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T06:59:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T06:59:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T06:59:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T06:59:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T06:59:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T06:59:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T06:59:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=12379], [object_number=66, tag_tid=14, entry_tid=12378], [object_number=65, tag_tid=14, entry_tid=12377], [object_number=64, tag_tid=14, entry_tid=12376]]], state=connected] 2024-09-14T06:59:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T06:59:59.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T06:59:59.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T06:59:59.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=12379' '!=' 'object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T06:59:59.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=12379' = 'object_number=66, tag_tid=14, entry_tid=12274' 2024-09-14T06:59:59.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T06:59:59.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:59.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:59.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12379 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12379 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T06:59:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T06:59:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T06:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T06:59:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T06:59:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T06:59:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T06:59:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T06:59:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T06:59:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T06:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T06:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:00.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:00.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:00.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:00.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12486], [object_number=65, tag_tid=14, entry_tid=12485], [object_number=64, tag_tid=14, entry_tid=12484], [object_number=67, tag_tid=14, entry_tid=12483]]], state=connected] 2024-09-14T07:00:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:00.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:00.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:00.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:00.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12486], [object_number=65, tag_tid=14, entry_tid=12485], [object_number=64, tag_tid=14, entry_tid=12484], [object_number=67, tag_tid=14, entry_tid=12483]]], state=connected] 2024-09-14T07:00:00.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12486' '!=' 'object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T07:00:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12486' = 'object_number=67, tag_tid=14, entry_tid=12379' 2024-09-14T07:00:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:00.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:00.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:00.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:00.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:00.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:00.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:00.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12486 2024-09-14T07:00:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12486 2024-09-14T07:00:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:00.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:00.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:00.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:00.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:00.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:00.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:00.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:00.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:00.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:00.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:00.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:00.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:00.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:00.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:00.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:00.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:00.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:00.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:00.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=12589], [object_number=64, tag_tid=14, entry_tid=12588], [object_number=67, tag_tid=14, entry_tid=12587], [object_number=66, tag_tid=14, entry_tid=12586]]], state=connected] 2024-09-14T07:00:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:00.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:00.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=12589], [object_number=64, tag_tid=14, entry_tid=12588], [object_number=67, tag_tid=14, entry_tid=12587], [object_number=66, tag_tid=14, entry_tid=12586]]], state=connected] 2024-09-14T07:00:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=12589' '!=' 'object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=12589' = 'object_number=66, tag_tid=14, entry_tid=12486' 2024-09-14T07:00:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:00.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:00.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:00.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:00.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:00.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:00.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:00.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:00.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:00.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:00.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12589 2024-09-14T07:00:00.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12589 2024-09-14T07:00:00.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:00.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:00.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:00.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:00.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:00.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:00.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:01.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:01.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:01.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12690], [object_number=65, tag_tid=14, entry_tid=12689], [object_number=64, tag_tid=14, entry_tid=12688], [object_number=67, tag_tid=14, entry_tid=12687]]], state=connected] 2024-09-14T07:00:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=12690], [object_number=65, tag_tid=14, entry_tid=12689], [object_number=64, tag_tid=14, entry_tid=12688], [object_number=67, tag_tid=14, entry_tid=12687]]], state=connected] 2024-09-14T07:00:01.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12690' '!=' 'object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=12690' = 'object_number=65, tag_tid=14, entry_tid=12589' 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:01.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12690 2024-09-14T07:00:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12690 2024-09-14T07:00:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:01.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:01.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:01.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:01.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:01.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:01.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:01.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:01.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:01.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:01.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:01.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:01.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:01.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:01.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:01.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:01.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:01.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:01.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:01.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:01.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:01.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:01.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:01.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:01.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:01.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:01.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:01.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:01.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:01.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:01.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:01.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:01.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:01.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:01.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:01.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=12799], [object_number=66, tag_tid=14, entry_tid=12798], [object_number=65, tag_tid=14, entry_tid=12797], [object_number=64, tag_tid=14, entry_tid=12796]]], state=connected] 2024-09-14T07:00:01.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:01.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:01.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:01.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=12799], [object_number=66, tag_tid=14, entry_tid=12798], [object_number=65, tag_tid=14, entry_tid=12797], [object_number=64, tag_tid=14, entry_tid=12796]]], state=connected] 2024-09-14T07:00:01.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:01.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:01.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:01.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=12799' '!=' 'object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:01.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=12799' = 'object_number=66, tag_tid=14, entry_tid=12690' 2024-09-14T07:00:01.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:01.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:01.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12799 2024-09-14T07:00:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12799 2024-09-14T07:00:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:01.718 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3807231282027128) > weight (0.25) 2024-09-14T07:00:01.718 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.776728618763675) > weight (0.25) 2024-09-14T07:00:01.718 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:00:01.718 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:00:01.719 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:00:01.719 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:00:01.718+0000 7f767555a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 30539) UID: 1000 2024-09-14T07:00:01.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:01.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:01.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:01.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:01.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:01.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:01.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:01.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:01.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:01.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:01.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:01.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:01.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:01.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:01.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:02.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:02.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:02.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:02.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:02.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:02.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:02.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=12900], [object_number=67, tag_tid=14, entry_tid=12899], [object_number=66, tag_tid=14, entry_tid=12898], [object_number=65, tag_tid=14, entry_tid=12897]]], state=connected] 2024-09-14T07:00:02.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:02.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:02.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:02.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:02.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:02.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:02.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=12900], [object_number=67, tag_tid=14, entry_tid=12899], [object_number=66, tag_tid=14, entry_tid=12898], [object_number=65, tag_tid=14, entry_tid=12897]]], state=connected] 2024-09-14T07:00:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=12900' '!=' 'object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=12900' = 'object_number=67, tag_tid=14, entry_tid=12799' 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:02.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:02.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:02.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:02.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:02.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:02.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12900 2024-09-14T07:00:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12900 2024-09-14T07:00:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:02.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:02.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:02.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:02.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:02.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:02.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:02.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:02.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:02.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:02.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:02.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:02.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:02.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:02.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=12995], [object_number=66, tag_tid=14, entry_tid=12994], [object_number=65, tag_tid=14, entry_tid=12993], [object_number=64, tag_tid=14, entry_tid=12992]]], state=connected] 2024-09-14T07:00:02.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:02.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:02.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:02.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:02.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=12995], [object_number=66, tag_tid=14, entry_tid=12994], [object_number=65, tag_tid=14, entry_tid=12993], [object_number=64, tag_tid=14, entry_tid=12992]]], state=connected] 2024-09-14T07:00:02.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=12995' '!=' 'object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=12995' = 'object_number=64, tag_tid=14, entry_tid=12900' 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:02.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:02.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:02.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:02.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:02.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:02.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:02.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:02.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:02.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:02.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:02.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:02.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12995 2024-09-14T07:00:02.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 12995 2024-09-14T07:00:02.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:02.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:02.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:02.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:02.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:02.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:02.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:02.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:02.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:02.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:02.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:03.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:03.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:03.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:03.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:03.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:03.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:03.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=13095], [object_number=66, tag_tid=14, entry_tid=13094], [object_number=65, tag_tid=14, entry_tid=13093], [object_number=64, tag_tid=14, entry_tid=13092]]], state=connected] 2024-09-14T07:00:03.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:03.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:03.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:03.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:03.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:03.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=13095], [object_number=66, tag_tid=14, entry_tid=13094], [object_number=65, tag_tid=14, entry_tid=13093], [object_number=64, tag_tid=14, entry_tid=13092]]], state=connected] 2024-09-14T07:00:03.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=13095' '!=' 'object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=13095' = 'object_number=67, tag_tid=14, entry_tid=12995' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13095 2024-09-14T07:00:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13095 2024-09-14T07:00:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:03.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:03.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:03.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:03.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:03.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:03.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:03.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:03.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:03.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:03.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:03.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:03.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:03.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:03.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:03.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:03.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:03.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=13216], [object_number=67, tag_tid=14, entry_tid=13215], [object_number=66, tag_tid=14, entry_tid=13214], [object_number=65, tag_tid=14, entry_tid=13213]]], state=connected] 2024-09-14T07:00:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=13216], [object_number=67, tag_tid=14, entry_tid=13215], [object_number=66, tag_tid=14, entry_tid=13214], [object_number=65, tag_tid=14, entry_tid=13213]]], state=connected] 2024-09-14T07:00:03.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=13216' '!=' 'object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=13216' = 'object_number=67, tag_tid=14, entry_tid=13095' 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:03.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:03.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:03.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13216 2024-09-14T07:00:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13216 2024-09-14T07:00:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:03.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:03.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:03.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:03.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:03.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:03.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=13331], [object_number=66, tag_tid=14, entry_tid=13330], [object_number=65, tag_tid=14, entry_tid=13329], [object_number=64, tag_tid=14, entry_tid=13328]]], state=connected] 2024-09-14T07:00:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:04.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=67, tag_tid=14, entry_tid=13331], [object_number=66, tag_tid=14, entry_tid=13330], [object_number=65, tag_tid=14, entry_tid=13329], [object_number=64, tag_tid=14, entry_tid=13328]]], state=connected] 2024-09-14T07:00:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=13331' '!=' 'object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=67, tag_tid=14, entry_tid=13331' = 'object_number=64, tag_tid=14, entry_tid=13216' 2024-09-14T07:00:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:04.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:04.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:04.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:04.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:04.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13331 2024-09-14T07:00:04.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13331 2024-09-14T07:00:04.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:04.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:04.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:04.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:04.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:04.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:04.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:04.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:04.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:04.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:04.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:04.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:04.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:04.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=13433], [object_number=64, tag_tid=14, entry_tid=13432], [object_number=67, tag_tid=14, entry_tid=13431], [object_number=66, tag_tid=14, entry_tid=13430]]], state=connected] 2024-09-14T07:00:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:04.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:04.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:04.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:04.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:04.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=13433], [object_number=64, tag_tid=14, entry_tid=13432], [object_number=67, tag_tid=14, entry_tid=13431], [object_number=66, tag_tid=14, entry_tid=13430]]], state=connected] 2024-09-14T07:00:04.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=13433' '!=' 'object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=13433' = 'object_number=67, tag_tid=14, entry_tid=13331' 2024-09-14T07:00:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:04.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:04.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:04.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:04.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:04.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:04.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:04.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13433 2024-09-14T07:00:04.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13433 2024-09-14T07:00:04.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:04.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:04.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:04.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:04.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:04.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:04.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:04.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:04.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:04.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:04.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:04.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:04.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:04.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:04.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:04.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:04.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=13533], [object_number=64, tag_tid=14, entry_tid=13532], [object_number=67, tag_tid=14, entry_tid=13531], [object_number=66, tag_tid=14, entry_tid=13530]]], state=connected] 2024-09-14T07:00:05.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:05.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:05.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:05.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:05.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=13533], [object_number=64, tag_tid=14, entry_tid=13532], [object_number=67, tag_tid=14, entry_tid=13531], [object_number=66, tag_tid=14, entry_tid=13530]]], state=connected] 2024-09-14T07:00:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=13533' '!=' 'object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=13533' = 'object_number=65, tag_tid=14, entry_tid=13433' 2024-09-14T07:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:05.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:05.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13533 2024-09-14T07:00:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13533 2024-09-14T07:00:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:05.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:05.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:05.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:05.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=13637], [object_number=64, tag_tid=14, entry_tid=13636], [object_number=67, tag_tid=14, entry_tid=13635], [object_number=66, tag_tid=14, entry_tid=13634]]], state=connected] 2024-09-14T07:00:05.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:05.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:05.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:05.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:05.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:05.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:05.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:05.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=13637], [object_number=64, tag_tid=14, entry_tid=13636], [object_number=67, tag_tid=14, entry_tid=13635], [object_number=66, tag_tid=14, entry_tid=13634]]], state=connected] 2024-09-14T07:00:05.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=13637' '!=' 'object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=13637' = 'object_number=65, tag_tid=14, entry_tid=13533' 2024-09-14T07:00:05.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:05.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:05.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:05.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:05.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:05.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:05.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:05.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13637 2024-09-14T07:00:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13637 2024-09-14T07:00:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:06.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:06.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:06.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:06.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:06.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:06.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:06.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:06.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:06.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:06.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:06.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:06.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:06.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=13792], [object_number=67, tag_tid=14, entry_tid=13791], [object_number=66, tag_tid=14, entry_tid=13790], [object_number=65, tag_tid=14, entry_tid=13789]]], state=connected] 2024-09-14T07:00:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:06.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:06.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:06.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:06.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:06.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:06.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:06.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=13792], [object_number=67, tag_tid=14, entry_tid=13791], [object_number=66, tag_tid=14, entry_tid=13790], [object_number=65, tag_tid=14, entry_tid=13789]]], state=connected] 2024-09-14T07:00:06.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=13792' '!=' 'object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:06.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=13792' = 'object_number=65, tag_tid=14, entry_tid=13637' 2024-09-14T07:00:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:06.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:06.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:06.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:06.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:06.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13792 2024-09-14T07:00:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13792 2024-09-14T07:00:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:06.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:06.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:06.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:06.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:06.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:06.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:06.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:06.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:06.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:06.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:06.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:06.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:06.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:06.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:06.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=13900], [object_number=67, tag_tid=14, entry_tid=13899], [object_number=66, tag_tid=14, entry_tid=13898], [object_number=65, tag_tid=14, entry_tid=13897]]], state=connected] 2024-09-14T07:00:06.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:06.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:06.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:06.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:06.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=64, tag_tid=14, entry_tid=13900], [object_number=67, tag_tid=14, entry_tid=13899], [object_number=66, tag_tid=14, entry_tid=13898], [object_number=65, tag_tid=14, entry_tid=13897]]], state=connected] 2024-09-14T07:00:06.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=13900' '!=' 'object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=64, tag_tid=14, entry_tid=13900' = 'object_number=64, tag_tid=14, entry_tid=13792' 2024-09-14T07:00:06.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:06.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:06.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:06.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:06.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:06.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:06.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:06.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:06.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:06.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:06.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:06.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:06.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:06.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13900 2024-09-14T07:00:06.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 13900 2024-09-14T07:00:06.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:06.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:06.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:06.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:06.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:06.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:06.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:06.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:06.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:06.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:06.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:06.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:06.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:06.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:07.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:07.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:07.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:07.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:07.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:07.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=14014], [object_number=65, tag_tid=14, entry_tid=14013], [object_number=64, tag_tid=14, entry_tid=14012], [object_number=67, tag_tid=14, entry_tid=14011]]], state=connected] 2024-09-14T07:00:07.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=14014], [object_number=65, tag_tid=14, entry_tid=14013], [object_number=64, tag_tid=14, entry_tid=14012], [object_number=67, tag_tid=14, entry_tid=14011]]], state=connected] 2024-09-14T07:00:07.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=14014' '!=' 'object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=14014' = 'object_number=64, tag_tid=14, entry_tid=13900' 2024-09-14T07:00:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:07.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:07.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:07.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:07.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:07.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:07.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:07.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:07.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14014 2024-09-14T07:00:07.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 14014 2024-09-14T07:00:07.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:07.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:07.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:07.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:07.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:07.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:07.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:07.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:07.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:07.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:07.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:07.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:07.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:07.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:07.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:07.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:07.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:07.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:07.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:07.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:07.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:07.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:07.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:07.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:07.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=14130], [object_number=65, tag_tid=14, entry_tid=14129], [object_number=64, tag_tid=14, entry_tid=14128], [object_number=67, tag_tid=14, entry_tid=14127]]], state=connected] 2024-09-14T07:00:07.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:07.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:07.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:07.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:07.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:07.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:07.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=14130], [object_number=65, tag_tid=14, entry_tid=14129], [object_number=64, tag_tid=14, entry_tid=14128], [object_number=67, tag_tid=14, entry_tid=14127]]], state=connected] 2024-09-14T07:00:07.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=14130' '!=' 'object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=14130' = 'object_number=66, tag_tid=14, entry_tid=14014' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:07.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:07.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14130 2024-09-14T07:00:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 14130 2024-09-14T07:00:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:07.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:07.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:07.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:08.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:08.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:08.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:08.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=14222], [object_number=65, tag_tid=14, entry_tid=14221], [object_number=64, tag_tid=14, entry_tid=14220], [object_number=67, tag_tid=14, entry_tid=14219]]], state=connected] 2024-09-14T07:00:08.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:08.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:08.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:08.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:08.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:08.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=66, tag_tid=14, entry_tid=14222], [object_number=65, tag_tid=14, entry_tid=14221], [object_number=64, tag_tid=14, entry_tid=14220], [object_number=67, tag_tid=14, entry_tid=14219]]], state=connected] 2024-09-14T07:00:08.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=14222' '!=' 'object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=66, tag_tid=14, entry_tid=14222' = 'object_number=66, tag_tid=14, entry_tid=14130' 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:08.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:08.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:08.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:08.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:08.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14222 2024-09-14T07:00:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 14222 2024-09-14T07:00:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:08.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:08.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:08.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:08.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:08.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:08.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:08.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:08.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:08.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:08.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:09.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:09.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=14281' '!=' 'object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:00:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=14281' = 'object_number=66, tag_tid=14, entry_tid=14222' 2024-09-14T07:00:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=15 2024-09-14T07:00:09.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:00:09.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:09.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=14 2024-09-14T07:00:09.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:09.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:09.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:00:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:00:09.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:00:09.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14281 2024-09-14T07:00:09.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 15 = 14 -a 1 -le 14281 2024-09-14T07:00:09.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:00:09.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:09.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:00:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:09.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:09.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:09.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:09.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:09.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:09.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:09.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:09.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:09.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:09.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:09.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:09.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=14281' '!=' 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:10.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:10.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:10.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:10.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:10.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:10.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:10.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:10.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:10.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:10.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:10.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:10.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:10.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:10.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:10.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:10.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=14281' '!=' 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:10.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:10.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:00:10.720 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:00:10.720 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:00:10.720 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:00:10.720 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:00:10.721 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:00:10.724 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:00:10.724 INFO:tasks.rbd_mirror_thrash:waiting for 82 secs before thrashing 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:11.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:11.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:11.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:11.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:11.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:11.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:11.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:11.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:11.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:11.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:11.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:11.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:11.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:11.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:11.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:11.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:11.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:11.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=14281' '!=' 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:11.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:11.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:00:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:13.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:13.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:13.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:13.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:13.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:13.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:13.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:13.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:13.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:13.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:13.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:13.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:13.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:13.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:13.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:13.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:13.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:13.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:13.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:13.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:13.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=14, entry_tid=14281], [object_number=64, tag_tid=14, entry_tid=14280], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:13.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=65, tag_tid=14, entry_tid=14281' '!=' 'object_number=65, tag_tid=14, entry_tid=14281' 2024-09-14T07:00:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:00:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:00:15.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:00:15.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:15.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:00:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:00:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:00:15.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:00:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:00:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:00:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:00:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:00:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:00:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:00:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:00:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:00:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:00:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:00:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:00:15.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:00:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:00:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:00:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:00:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:15.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:15.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:15.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:15.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:15.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:00:15.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:15.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:00:15.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:00:15.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:15.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:00:15.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:15.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 16 2024-09-14T07:00:15.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 16 2024-09-14T07:00:15.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:00:15.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:15.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=65, tag_tid=15, entry_tid=1], [object_number=64, tag_tid=15, entry_tid=0], [object_number=67, tag_tid=14, entry_tid=14279], [object_number=66, tag_tid=14, entry_tid=14278]]], state=connected] 2024-09-14T07:00:15.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:00:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=65, tag_tid=15, entry_tid=1' -a 'object_number=65, tag_tid=15, entry_tid=1' = 'object_number=65, tag_tid=15, entry_tid=1' 2024-09-14T07:00:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:00:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:00:16.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 9 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:00:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 06:58:01 2024 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:00:00 2024 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:00:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap4-clean ']' 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap4-clean 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4-clean 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:00:17.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:00:17.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:00:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:18.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap5 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:18.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:18.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:00:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:00:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:20.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:00:20.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:21.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap5-clean 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap5-clean 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5-clean 2024-09-14T07:00:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T07:00:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:00:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T07:00:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:00:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:21.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:00:21.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:00:21.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:00:21.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T07:00:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 78543872 bytes 2024-09-14T07:00:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 181568512 bytes 2024-09-14T07:00:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 197878784 bytes 2024-09-14T07:00:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:00:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:00:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:00:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:00:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:00:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:00:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:00:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T07:00:30.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T07:00:30.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:00:30.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T07:00:30.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":3170292.27,"entries_behind_primary":1,"entries_per_second":201.03,"non_primary_position":{"entry_tid":14281,"object_number":65,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":65,"tag_tid":15},"seconds_until_synced":0} 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:00:10 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:00:10' 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T07:00:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T07:00:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T07:00:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":3170292.27,"entries_behind_primary":1,"entries_per_second":201.03,"non_primary_position":{"entry_tid":14281,"object_number":65,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":65,"tag_tid":15},"seconds_until_synced":0} 2024-09-14T07:00:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:00:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:00:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T07:00:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:00:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T07:00:30.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T07:00:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T07:00:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T07:00:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:00:10' 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 07:00:10' 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 07:00:10' 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:00:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T07:00:30.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589trueOK43674370mirror.1smithi16919.3.0-4775-g7f24e589falseOK44074410mirror.2smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":3170292.27,"entries_behind_primary":1,"entries_per_second":201.03,"non_primary_position":{"entry_tid":14281,"object_number":65,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":65,"tag_tid":15},"seconds_until_synced":0}42104261mirror.0smithi1692024-09-14 07:00:10cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 07:00:10' 2024-09-14T07:00:30.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T07:00:30.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T07:00:30.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:00:10' 2024-09-14T07:00:30.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T07:00:30.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:00:30.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:30.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:00:30.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap6 2024-09-14T07:00:30.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap6 2024-09-14T07:00:30.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:00:30.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:00:30.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:00:30.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6 2024-09-14T07:00:30.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:00:33.721 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.41779177411717106) > weight (0.25) 2024-09-14T07:00:33.722 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T07:00:33.722 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T07:00:33.722 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T07:00:33.723 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:00:33.721+0000 7fe94cad8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 45722) UID: 1000 2024-09-14T07:00:37.723 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T07:00:37.724 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:00:37.724 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T07:00:37.724 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T07:00:37.724 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T07:00:37.727 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T07:00:37.727 INFO:tasks.rbd_mirror_thrash:waiting for 106 secs before thrashing 2024-09-14T07:01:01.245 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:01:01.245+0000 7f99ede58640 -1 reset not still connected to 0x5592105aec30 2024-09-14T07:01:01.246 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:01:01.249+0000 7f99ede58640 -1 reset not still connected to 0x5592115e49c0 2024-09-14T07:01:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T07:01:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:01.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:01:02.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:02.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:01:02.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:02.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:01:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:01:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:01:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:01:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:02.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:02.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:01:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:02.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:02.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:02.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:01:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:01:02.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:01:02.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:01:02.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:01:02.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:01:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:02.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:02.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:02.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:02.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:02.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:02.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:02.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:03.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:03.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:03.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7451], [object_number=74, tag_tid=17, entry_tid=7450], [object_number=73, tag_tid=17, entry_tid=7449], [object_number=72, tag_tid=17, entry_tid=7448]]], state=connected] 2024-09-14T07:01:03.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:03.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:03.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:03.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:03.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7451], [object_number=74, tag_tid=17, entry_tid=7450], [object_number=73, tag_tid=17, entry_tid=7449], [object_number=72, tag_tid=17, entry_tid=7448]]], state=connected] 2024-09-14T07:01:03.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7451' '!=' '' 2024-09-14T07:01:03.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:03.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7451' = '' 2024-09-14T07:01:03.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:03.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:03.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:03.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:03.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:03.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:03.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:03.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7451 2024-09-14T07:01:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 7451 2024-09-14T07:01:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:03.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:03.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:03.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:03.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:03.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:03.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:03.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7539], [object_number=74, tag_tid=17, entry_tid=7538], [object_number=73, tag_tid=17, entry_tid=7537], [object_number=72, tag_tid=17, entry_tid=7536]]], state=connected] 2024-09-14T07:01:03.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:03.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:03.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:03.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:03.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:03.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7539], [object_number=74, tag_tid=17, entry_tid=7538], [object_number=73, tag_tid=17, entry_tid=7537], [object_number=72, tag_tid=17, entry_tid=7536]]], state=connected] 2024-09-14T07:01:03.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:03.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:03.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:03.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7539' '!=' 'object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:03.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7539' = 'object_number=75, tag_tid=17, entry_tid=7451' 2024-09-14T07:01:03.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:03.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:03.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:03.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:03.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:03.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:03.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:03.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:03.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:03.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:03.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:03.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:03.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7539 2024-09-14T07:01:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 7539 2024-09-14T07:01:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:03.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:03.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:03.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:03.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:03.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:03.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:03.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:03.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:03.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:03.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:04.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:04.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:04.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:04.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=7634], [object_number=73, tag_tid=17, entry_tid=7633], [object_number=72, tag_tid=17, entry_tid=7632], [object_number=75, tag_tid=17, entry_tid=7631]]], state=connected] 2024-09-14T07:01:04.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:04.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:04.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:04.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:04.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:04.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:04.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=7634], [object_number=73, tag_tid=17, entry_tid=7633], [object_number=72, tag_tid=17, entry_tid=7632], [object_number=75, tag_tid=17, entry_tid=7631]]], state=connected] 2024-09-14T07:01:04.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=7634' '!=' 'object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=7634' = 'object_number=75, tag_tid=17, entry_tid=7539' 2024-09-14T07:01:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:04.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:04.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:04.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:04.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:04.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:04.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:04.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7634 2024-09-14T07:01:04.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 7634 2024-09-14T07:01:04.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:04.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:04.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:04.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:04.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:04.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:04.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:04.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:04.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:04.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:04.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:04.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:04.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:04.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:04.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:04.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:04.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7735], [object_number=74, tag_tid=17, entry_tid=7734], [object_number=73, tag_tid=17, entry_tid=7733], [object_number=72, tag_tid=17, entry_tid=7732]]], state=connected] 2024-09-14T07:01:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:04.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:04.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7735], [object_number=74, tag_tid=17, entry_tid=7734], [object_number=73, tag_tid=17, entry_tid=7733], [object_number=72, tag_tid=17, entry_tid=7732]]], state=connected] 2024-09-14T07:01:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7735' '!=' 'object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7735' = 'object_number=74, tag_tid=17, entry_tid=7634' 2024-09-14T07:01:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:04.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:04.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:04.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:04.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:04.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:04.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:04.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7735 2024-09-14T07:01:04.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 7735 2024-09-14T07:01:04.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:04.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:04.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:04.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:04.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:04.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:05.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:05.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:05.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:05.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:05.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:05.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:05.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:05.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:05.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:05.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7839], [object_number=74, tag_tid=17, entry_tid=7838], [object_number=73, tag_tid=17, entry_tid=7837], [object_number=72, tag_tid=17, entry_tid=7836]]], state=connected] 2024-09-14T07:01:05.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:05.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:05.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:05.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:05.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:05.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:05.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7839], [object_number=74, tag_tid=17, entry_tid=7838], [object_number=73, tag_tid=17, entry_tid=7837], [object_number=72, tag_tid=17, entry_tid=7836]]], state=connected] 2024-09-14T07:01:05.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7839' '!=' 'object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7839' = 'object_number=75, tag_tid=17, entry_tid=7735' 2024-09-14T07:01:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:05.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:05.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:05.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:05.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:05.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7839 2024-09-14T07:01:05.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 7839 2024-09-14T07:01:05.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:05.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:05.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:05.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:05.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:05.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:05.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:05.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:05.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:05.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:05.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:05.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:05.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:05.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:05.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:05.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:05.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:05.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:05.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:05.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:05.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7947], [object_number=74, tag_tid=17, entry_tid=7946], [object_number=73, tag_tid=17, entry_tid=7945], [object_number=72, tag_tid=17, entry_tid=7944]]], state=connected] 2024-09-14T07:01:05.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:05.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:05.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:05.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:05.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:05.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=7947], [object_number=74, tag_tid=17, entry_tid=7946], [object_number=73, tag_tid=17, entry_tid=7945], [object_number=72, tag_tid=17, entry_tid=7944]]], state=connected] 2024-09-14T07:01:05.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7947' '!=' 'object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=7947' = 'object_number=75, tag_tid=17, entry_tid=7839' 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:05.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:05.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:05.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:05.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:05.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:05.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:05.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:05.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:05.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:05.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7947 2024-09-14T07:01:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 7947 2024-09-14T07:01:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:05.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:05.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:05.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:05.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:05.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:05.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:05.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:05.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:06.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:06.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:06.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:06.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:06.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:06.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:06.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:06.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=8038], [object_number=73, tag_tid=17, entry_tid=8037], [object_number=72, tag_tid=17, entry_tid=8036], [object_number=75, tag_tid=17, entry_tid=8035]]], state=connected] 2024-09-14T07:01:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:06.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=8038], [object_number=73, tag_tid=17, entry_tid=8037], [object_number=72, tag_tid=17, entry_tid=8036], [object_number=75, tag_tid=17, entry_tid=8035]]], state=connected] 2024-09-14T07:01:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=8038' '!=' 'object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=8038' = 'object_number=75, tag_tid=17, entry_tid=7947' 2024-09-14T07:01:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:06.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:06.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:06.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:06.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:06.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:06.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8038 2024-09-14T07:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8038 2024-09-14T07:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:06.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:06.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:06.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:06.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:06.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8127], [object_number=74, tag_tid=17, entry_tid=8126], [object_number=73, tag_tid=17, entry_tid=8125], [object_number=72, tag_tid=17, entry_tid=8124]]], state=connected] 2024-09-14T07:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:06.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:06.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:06.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8127], [object_number=74, tag_tid=17, entry_tid=8126], [object_number=73, tag_tid=17, entry_tid=8125], [object_number=72, tag_tid=17, entry_tid=8124]]], state=connected] 2024-09-14T07:01:06.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8127' '!=' 'object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8127' = 'object_number=74, tag_tid=17, entry_tid=8038' 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:06.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:06.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:06.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8127 2024-09-14T07:01:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8127 2024-09-14T07:01:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:06.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:06.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:06.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:06.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:06.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:06.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:06.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:06.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:06.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:06.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:07.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:07.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:07.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:07.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:07.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=72, tag_tid=17, entry_tid=8212], [object_number=75, tag_tid=17, entry_tid=8211], [object_number=74, tag_tid=17, entry_tid=8210], [object_number=73, tag_tid=17, entry_tid=8209]]], state=connected] 2024-09-14T07:01:07.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:07.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:07.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:07.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:07.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:07.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=72, tag_tid=17, entry_tid=8212], [object_number=75, tag_tid=17, entry_tid=8211], [object_number=74, tag_tid=17, entry_tid=8210], [object_number=73, tag_tid=17, entry_tid=8209]]], state=connected] 2024-09-14T07:01:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=72, tag_tid=17, entry_tid=8212' '!=' 'object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:07.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:07.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=72, tag_tid=17, entry_tid=8212' = 'object_number=75, tag_tid=17, entry_tid=8127' 2024-09-14T07:01:07.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:07.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:07.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:07.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:07.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:07.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:07.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:07.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:07.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8212 2024-09-14T07:01:07.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8212 2024-09-14T07:01:07.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:07.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:07.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:07.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:07.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:07.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:07.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:07.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:07.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:07.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8351], [object_number=74, tag_tid=17, entry_tid=8350], [object_number=73, tag_tid=17, entry_tid=8349], [object_number=72, tag_tid=17, entry_tid=8348]]], state=connected] 2024-09-14T07:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:07.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:07.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:07.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:07.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:07.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8351], [object_number=74, tag_tid=17, entry_tid=8350], [object_number=73, tag_tid=17, entry_tid=8349], [object_number=72, tag_tid=17, entry_tid=8348]]], state=connected] 2024-09-14T07:01:07.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:07.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8351' '!=' 'object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8351' = 'object_number=72, tag_tid=17, entry_tid=8212' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8351 2024-09-14T07:01:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8351 2024-09-14T07:01:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:08.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:08.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:08.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:08.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:08.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:08.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:08.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:08.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:08.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:08.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8463], [object_number=74, tag_tid=17, entry_tid=8462], [object_number=73, tag_tid=17, entry_tid=8461], [object_number=72, tag_tid=17, entry_tid=8460]]], state=connected] 2024-09-14T07:01:08.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:08.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:08.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:08.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:08.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:08.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:08.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:08.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:08.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8463], [object_number=74, tag_tid=17, entry_tid=8462], [object_number=73, tag_tid=17, entry_tid=8461], [object_number=72, tag_tid=17, entry_tid=8460]]], state=connected] 2024-09-14T07:01:08.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8463' '!=' 'object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8463' = 'object_number=75, tag_tid=17, entry_tid=8351' 2024-09-14T07:01:08.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:08.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:08.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:08.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:08.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:08.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:08.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:08.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:08.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:08.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:08.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:08.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:08.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8463 2024-09-14T07:01:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8463 2024-09-14T07:01:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:08.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:08.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:08.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:09.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:09.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:09.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:09.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:09.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:09.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=8618], [object_number=73, tag_tid=17, entry_tid=8617], [object_number=72, tag_tid=17, entry_tid=8616], [object_number=75, tag_tid=17, entry_tid=8615]]], state=connected] 2024-09-14T07:01:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:09.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:09.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:09.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:09.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:09.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:09.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:09.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=8618], [object_number=73, tag_tid=17, entry_tid=8617], [object_number=72, tag_tid=17, entry_tid=8616], [object_number=75, tag_tid=17, entry_tid=8615]]], state=connected] 2024-09-14T07:01:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=8618' '!=' 'object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=8618' = 'object_number=75, tag_tid=17, entry_tid=8463' 2024-09-14T07:01:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:09.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:09.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:09.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8618 2024-09-14T07:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8618 2024-09-14T07:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:09.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:09.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:09.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:09.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:09.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:09.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:09.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:09.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:09.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:09.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:09.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:09.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8751], [object_number=74, tag_tid=17, entry_tid=8750], [object_number=73, tag_tid=17, entry_tid=8749], [object_number=72, tag_tid=17, entry_tid=8748]]], state=connected] 2024-09-14T07:01:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:09.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:09.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:09.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:09.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:09.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:09.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:09.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:09.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:09.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:09.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=8751], [object_number=74, tag_tid=17, entry_tid=8750], [object_number=73, tag_tid=17, entry_tid=8749], [object_number=72, tag_tid=17, entry_tid=8748]]], state=connected] 2024-09-14T07:01:09.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8751' '!=' 'object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=8751' = 'object_number=74, tag_tid=17, entry_tid=8618' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:09.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:09.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:09.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:09.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:09.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:09.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8751 2024-09-14T07:01:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8751 2024-09-14T07:01:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:09.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:09.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:09.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:09.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:09.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:09.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:10.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:10.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:10.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:10.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:10.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:10.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:10.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=73, tag_tid=17, entry_tid=8865], [object_number=72, tag_tid=17, entry_tid=8864], [object_number=75, tag_tid=17, entry_tid=8863], [object_number=74, tag_tid=17, entry_tid=8862]]], state=connected] 2024-09-14T07:01:10.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=73, tag_tid=17, entry_tid=8865], [object_number=72, tag_tid=17, entry_tid=8864], [object_number=75, tag_tid=17, entry_tid=8863], [object_number=74, tag_tid=17, entry_tid=8862]]], state=connected] 2024-09-14T07:01:10.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=73, tag_tid=17, entry_tid=8865' '!=' 'object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=73, tag_tid=17, entry_tid=8865' = 'object_number=75, tag_tid=17, entry_tid=8751' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8865 2024-09-14T07:01:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8865 2024-09-14T07:01:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:10.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:10.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:10.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:10.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:10.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:10.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:10.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:10.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=8986], [object_number=73, tag_tid=17, entry_tid=8985], [object_number=72, tag_tid=17, entry_tid=8984], [object_number=75, tag_tid=17, entry_tid=8983]]], state=connected] 2024-09-14T07:01:10.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:10.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:10.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:10.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:10.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:10.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:10.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:10.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:10.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:10.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:10.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:10.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:10.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=8986], [object_number=73, tag_tid=17, entry_tid=8985], [object_number=72, tag_tid=17, entry_tid=8984], [object_number=75, tag_tid=17, entry_tid=8983]]], state=connected] 2024-09-14T07:01:10.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=8986' '!=' 'object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=8986' = 'object_number=73, tag_tid=17, entry_tid=8865' 2024-09-14T07:01:10.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8986 2024-09-14T07:01:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 8986 2024-09-14T07:01:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:10.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:10.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:10.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:11.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:11.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:11.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:11.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=73, tag_tid=17, entry_tid=9085], [object_number=72, tag_tid=17, entry_tid=9084], [object_number=75, tag_tid=17, entry_tid=9083], [object_number=74, tag_tid=17, entry_tid=9082]]], state=connected] 2024-09-14T07:01:11.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:11.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:11.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:11.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=73, tag_tid=17, entry_tid=9085], [object_number=72, tag_tid=17, entry_tid=9084], [object_number=75, tag_tid=17, entry_tid=9083], [object_number=74, tag_tid=17, entry_tid=9082]]], state=connected] 2024-09-14T07:01:11.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=73, tag_tid=17, entry_tid=9085' '!=' 'object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=73, tag_tid=17, entry_tid=9085' = 'object_number=74, tag_tid=17, entry_tid=8986' 2024-09-14T07:01:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:11.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:11.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:11.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:11.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:11.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9085 2024-09-14T07:01:11.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9085 2024-09-14T07:01:11.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:11.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:11.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:11.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:11.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:11.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:11.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:11.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:11.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:11.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:11.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=9182], [object_number=73, tag_tid=17, entry_tid=9181], [object_number=72, tag_tid=17, entry_tid=9180], [object_number=75, tag_tid=17, entry_tid=9179]]], state=connected] 2024-09-14T07:01:11.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:11.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=9182], [object_number=73, tag_tid=17, entry_tid=9181], [object_number=72, tag_tid=17, entry_tid=9180], [object_number=75, tag_tid=17, entry_tid=9179]]], state=connected] 2024-09-14T07:01:11.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=9182' '!=' 'object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=9182' = 'object_number=73, tag_tid=17, entry_tid=9085' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:11.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:11.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9182 2024-09-14T07:01:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9182 2024-09-14T07:01:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:11.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:11.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:11.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:11.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:11.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:11.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:11.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:11.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:11.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:11.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:11.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:11.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:11.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:11.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:11.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:12.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:12.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=9235], [object_number=74, tag_tid=17, entry_tid=9234], [object_number=73, tag_tid=17, entry_tid=9233], [object_number=72, tag_tid=17, entry_tid=9232]]], state=connected] 2024-09-14T07:01:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:12.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:12.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:12.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:12.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:12.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:12.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=9235], [object_number=74, tag_tid=17, entry_tid=9234], [object_number=73, tag_tid=17, entry_tid=9233], [object_number=72, tag_tid=17, entry_tid=9232]]], state=connected] 2024-09-14T07:01:12.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=9235' '!=' 'object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=9235' = 'object_number=74, tag_tid=17, entry_tid=9182' 2024-09-14T07:01:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:12.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9235 2024-09-14T07:01:12.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9235 2024-09-14T07:01:12.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:12.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:12.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:12.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:12.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:12.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:12.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:12.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:12.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:12.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:12.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:12.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:12.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:12.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=9330], [object_number=73, tag_tid=17, entry_tid=9329], [object_number=72, tag_tid=17, entry_tid=9328], [object_number=75, tag_tid=17, entry_tid=9327]]], state=connected] 2024-09-14T07:01:12.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=9330], [object_number=73, tag_tid=17, entry_tid=9329], [object_number=72, tag_tid=17, entry_tid=9328], [object_number=75, tag_tid=17, entry_tid=9327]]], state=connected] 2024-09-14T07:01:12.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=9330' '!=' 'object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=9330' = 'object_number=75, tag_tid=17, entry_tid=9235' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:12.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:12.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:12.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:12.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:12.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:12.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:12.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:12.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:12.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9330 2024-09-14T07:01:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9330 2024-09-14T07:01:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:12.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:12.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:12.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:12.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:12.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:12.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:12.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:12.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=73, tag_tid=17, entry_tid=9425], [object_number=72, tag_tid=17, entry_tid=9424], [object_number=75, tag_tid=17, entry_tid=9423], [object_number=74, tag_tid=17, entry_tid=9422]]], state=connected] 2024-09-14T07:01:13.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:13.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:13.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:13.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:13.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:13.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:13.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=73, tag_tid=17, entry_tid=9425], [object_number=72, tag_tid=17, entry_tid=9424], [object_number=75, tag_tid=17, entry_tid=9423], [object_number=74, tag_tid=17, entry_tid=9422]]], state=connected] 2024-09-14T07:01:13.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=73, tag_tid=17, entry_tid=9425' '!=' 'object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=73, tag_tid=17, entry_tid=9425' = 'object_number=74, tag_tid=17, entry_tid=9330' 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:13.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:13.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:13.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:13.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:13.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:13.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9425 2024-09-14T07:01:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9425 2024-09-14T07:01:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:13.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:13.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:13.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:13.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:13.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:13.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:13.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:13.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:13.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:13.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:13.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=72, tag_tid=17, entry_tid=9524], [object_number=75, tag_tid=17, entry_tid=9523], [object_number=74, tag_tid=17, entry_tid=9522], [object_number=73, tag_tid=17, entry_tid=9521]]], state=connected] 2024-09-14T07:01:13.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:13.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=72, tag_tid=17, entry_tid=9524], [object_number=75, tag_tid=17, entry_tid=9523], [object_number=74, tag_tid=17, entry_tid=9522], [object_number=73, tag_tid=17, entry_tid=9521]]], state=connected] 2024-09-14T07:01:13.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=72, tag_tid=17, entry_tid=9524' '!=' 'object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=72, tag_tid=17, entry_tid=9524' = 'object_number=73, tag_tid=17, entry_tid=9425' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:13.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:13.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:13.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:13.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:13.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:13.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:13.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:13.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:13.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:13.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9524 2024-09-14T07:01:13.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9524 2024-09-14T07:01:13.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:13.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:13.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:13.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:13.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:13.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:13.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:13.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:13.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:14.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:14.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:14.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=9623], [object_number=74, tag_tid=17, entry_tid=9622], [object_number=73, tag_tid=17, entry_tid=9621], [object_number=72, tag_tid=17, entry_tid=9620]]], state=connected] 2024-09-14T07:01:14.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:14.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:14.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=75, tag_tid=17, entry_tid=9623], [object_number=74, tag_tid=17, entry_tid=9622], [object_number=73, tag_tid=17, entry_tid=9621], [object_number=72, tag_tid=17, entry_tid=9620]]], state=connected] 2024-09-14T07:01:14.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=9623' '!=' 'object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=75, tag_tid=17, entry_tid=9623' = 'object_number=72, tag_tid=17, entry_tid=9524' 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:14.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:14.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:14.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:14.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:14.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:14.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:14.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:14.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:14.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9623 2024-09-14T07:01:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9623 2024-09-14T07:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:14.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:14.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:14.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:14.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:14.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:14.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:14.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:14.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:14.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:14.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=9726], [object_number=73, tag_tid=17, entry_tid=9725], [object_number=72, tag_tid=17, entry_tid=9724], [object_number=75, tag_tid=17, entry_tid=9723]]], state=connected] 2024-09-14T07:01:14.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:14.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:14.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:14.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:14.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 18 2024-09-14T07:01:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=74, tag_tid=17, entry_tid=9726], [object_number=73, tag_tid=17, entry_tid=9725], [object_number=72, tag_tid=17, entry_tid=9724], [object_number=75, tag_tid=17, entry_tid=9723]]], state=connected] 2024-09-14T07:01:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=9726' '!=' 'object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=74, tag_tid=17, entry_tid=9726' = 'object_number=75, tag_tid=17, entry_tid=9623' 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:14.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:14.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:14.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:14.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:14.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:14.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:14.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:14.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:14.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:14.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:14.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:14.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9726 2024-09-14T07:01:14.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9726 2024-09-14T07:01:14.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:14.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:14.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:14.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:14.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:14.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:14.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:14.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:14.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=9802], [object_number=77, tag_tid=17, entry_tid=9801], [object_number=76, tag_tid=17, entry_tid=9800], [object_number=79, tag_tid=17, entry_tid=9799]]], state=connected] 2024-09-14T07:01:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:15.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:15.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:15.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:15.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:15.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:15.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=9802], [object_number=77, tag_tid=17, entry_tid=9801], [object_number=76, tag_tid=17, entry_tid=9800], [object_number=79, tag_tid=17, entry_tid=9799]]], state=connected] 2024-09-14T07:01:15.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=9802' '!=' 'object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:15.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=9802' = 'object_number=74, tag_tid=17, entry_tid=9726' 2024-09-14T07:01:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:15.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:15.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:15.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:15.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:15.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9802 2024-09-14T07:01:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9802 2024-09-14T07:01:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:15.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:15.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:15.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:15.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:15.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:15.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:15.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:15.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:15.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:15.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:15.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:15.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:15.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=9905], [object_number=76, tag_tid=17, entry_tid=9904], [object_number=79, tag_tid=17, entry_tid=9903], [object_number=78, tag_tid=17, entry_tid=9902]]], state=connected] 2024-09-14T07:01:15.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:15.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:15.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:15.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:15.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=9905], [object_number=76, tag_tid=17, entry_tid=9904], [object_number=79, tag_tid=17, entry_tid=9903], [object_number=78, tag_tid=17, entry_tid=9902]]], state=connected] 2024-09-14T07:01:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:15.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:15.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=9905' '!=' 'object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=9905' = 'object_number=78, tag_tid=17, entry_tid=9802' 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:15.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:15.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:15.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:15.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:15.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:15.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:15.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9905 2024-09-14T07:01:15.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 9905 2024-09-14T07:01:15.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:15.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:15.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:15.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:15.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:15.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:15.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:15.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:15.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:15.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:15.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:15.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:15.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:15.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:15.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:15.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:15.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:15.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:15.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10004], [object_number=79, tag_tid=17, entry_tid=10003], [object_number=78, tag_tid=17, entry_tid=10002], [object_number=77, tag_tid=17, entry_tid=10001]]], state=connected] 2024-09-14T07:01:16.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:16.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:16.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10004], [object_number=79, tag_tid=17, entry_tid=10003], [object_number=78, tag_tid=17, entry_tid=10002], [object_number=77, tag_tid=17, entry_tid=10001]]], state=connected] 2024-09-14T07:01:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10004' '!=' 'object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10004' = 'object_number=77, tag_tid=17, entry_tid=9905' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:16.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:16.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10004 2024-09-14T07:01:16.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10004 2024-09-14T07:01:16.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:16.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:16.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:16.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:16.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:16.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:16.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:16.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:16.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:16.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:16.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:16.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:16.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:16.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:16.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:16.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:16.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10112], [object_number=79, tag_tid=17, entry_tid=10111], [object_number=78, tag_tid=17, entry_tid=10110], [object_number=77, tag_tid=17, entry_tid=10109]]], state=connected] 2024-09-14T07:01:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10112], [object_number=79, tag_tid=17, entry_tid=10111], [object_number=78, tag_tid=17, entry_tid=10110], [object_number=77, tag_tid=17, entry_tid=10109]]], state=connected] 2024-09-14T07:01:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10112' '!=' 'object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10112' = 'object_number=76, tag_tid=17, entry_tid=10004' 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:16.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:16.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:16.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:16.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10112 2024-09-14T07:01:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10112 2024-09-14T07:01:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:16.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:16.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:16.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:16.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:16.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:16.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:16.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10232], [object_number=79, tag_tid=17, entry_tid=10231], [object_number=78, tag_tid=17, entry_tid=10230], [object_number=77, tag_tid=17, entry_tid=10229]]], state=connected] 2024-09-14T07:01:16.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:16.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:16.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:16.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:16.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:16.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10232], [object_number=79, tag_tid=17, entry_tid=10231], [object_number=78, tag_tid=17, entry_tid=10230], [object_number=77, tag_tid=17, entry_tid=10229]]], state=connected] 2024-09-14T07:01:17.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10232' '!=' 'object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10232' = 'object_number=76, tag_tid=17, entry_tid=10112' 2024-09-14T07:01:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:17.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:17.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:17.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:17.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:17.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:17.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:17.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:17.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:17.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10232 2024-09-14T07:01:17.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10232 2024-09-14T07:01:17.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:17.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:17.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:17.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:17.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:17.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:17.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:17.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:17.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:17.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:17.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:17.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:17.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=10342], [object_number=77, tag_tid=17, entry_tid=10341], [object_number=76, tag_tid=17, entry_tid=10340], [object_number=79, tag_tid=17, entry_tid=10339]]], state=connected] 2024-09-14T07:01:17.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:17.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:17.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:17.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:17.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=10342], [object_number=77, tag_tid=17, entry_tid=10341], [object_number=76, tag_tid=17, entry_tid=10340], [object_number=79, tag_tid=17, entry_tid=10339]]], state=connected] 2024-09-14T07:01:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=10342' '!=' 'object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=10342' = 'object_number=76, tag_tid=17, entry_tid=10232' 2024-09-14T07:01:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:17.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:17.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:17.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:17.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10342 2024-09-14T07:01:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10342 2024-09-14T07:01:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:17.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:17.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:17.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10456], [object_number=79, tag_tid=17, entry_tid=10455], [object_number=78, tag_tid=17, entry_tid=10454], [object_number=77, tag_tid=17, entry_tid=10453]]], state=connected] 2024-09-14T07:01:17.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:17.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:18.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:18.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:18.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:18.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10456], [object_number=79, tag_tid=17, entry_tid=10455], [object_number=78, tag_tid=17, entry_tid=10454], [object_number=77, tag_tid=17, entry_tid=10453]]], state=connected] 2024-09-14T07:01:18.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10456' '!=' 'object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10456' = 'object_number=78, tag_tid=17, entry_tid=10342' 2024-09-14T07:01:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:18.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:18.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:18.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:18.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:18.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:18.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:18.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:18.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:18.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10456 2024-09-14T07:01:18.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10456 2024-09-14T07:01:18.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:18.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:18.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:18.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:18.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:18.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:18.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:18.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:18.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:18.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:18.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:18.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:18.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:18.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:18.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:18.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:18.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:18.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:18.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=10561], [object_number=76, tag_tid=17, entry_tid=10560], [object_number=79, tag_tid=17, entry_tid=10559], [object_number=78, tag_tid=17, entry_tid=10558]]], state=connected] 2024-09-14T07:01:18.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:18.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:18.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:18.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:18.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:18.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:18.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=10561], [object_number=76, tag_tid=17, entry_tid=10560], [object_number=79, tag_tid=17, entry_tid=10559], [object_number=78, tag_tid=17, entry_tid=10558]]], state=connected] 2024-09-14T07:01:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=10561' '!=' 'object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=10561' = 'object_number=76, tag_tid=17, entry_tid=10456' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10561 2024-09-14T07:01:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10561 2024-09-14T07:01:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:18.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:18.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:18.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:18.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:18.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:18.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:18.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:18.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:18.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:18.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:18.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:18.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:18.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:18.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:18.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:18.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:18.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:18.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:18.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=10654], [object_number=77, tag_tid=17, entry_tid=10653], [object_number=76, tag_tid=17, entry_tid=10652], [object_number=79, tag_tid=17, entry_tid=10651]]], state=connected] 2024-09-14T07:01:18.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:18.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:18.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:18.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:18.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:19.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:19.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:19.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:19.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=10654], [object_number=77, tag_tid=17, entry_tid=10653], [object_number=76, tag_tid=17, entry_tid=10652], [object_number=79, tag_tid=17, entry_tid=10651]]], state=connected] 2024-09-14T07:01:19.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=10654' '!=' 'object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=10654' = 'object_number=77, tag_tid=17, entry_tid=10561' 2024-09-14T07:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:19.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:19.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:19.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:19.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:19.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10654 2024-09-14T07:01:19.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10654 2024-09-14T07:01:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:19.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:19.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:19.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:19.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:19.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:19.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:19.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:19.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:19.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:19.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:19.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:19.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:19.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:19.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:19.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=10745], [object_number=76, tag_tid=17, entry_tid=10744], [object_number=79, tag_tid=17, entry_tid=10743], [object_number=78, tag_tid=17, entry_tid=10742]]], state=connected] 2024-09-14T07:01:19.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:19.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:19.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:19.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:19.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=10745], [object_number=76, tag_tid=17, entry_tid=10744], [object_number=79, tag_tid=17, entry_tid=10743], [object_number=78, tag_tid=17, entry_tid=10742]]], state=connected] 2024-09-14T07:01:19.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=10745' '!=' 'object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:19.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=10745' = 'object_number=78, tag_tid=17, entry_tid=10654' 2024-09-14T07:01:19.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:19.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:19.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:19.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:19.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10745 2024-09-14T07:01:19.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10745 2024-09-14T07:01:19.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:19.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:19.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:19.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:19.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:19.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:19.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:19.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:19.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:19.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:19.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:19.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:19.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:19.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:19.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:19.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=10843], [object_number=78, tag_tid=17, entry_tid=10842], [object_number=77, tag_tid=17, entry_tid=10841], [object_number=76, tag_tid=17, entry_tid=10840]]], state=connected] 2024-09-14T07:01:19.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=10843], [object_number=78, tag_tid=17, entry_tid=10842], [object_number=77, tag_tid=17, entry_tid=10841], [object_number=76, tag_tid=17, entry_tid=10840]]], state=connected] 2024-09-14T07:01:19.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:19.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=10843' '!=' 'object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=10843' = 'object_number=77, tag_tid=17, entry_tid=10745' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:19.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:19.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:19.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10843 2024-09-14T07:01:20.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10843 2024-09-14T07:01:20.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:20.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:20.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:20.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:20.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:20.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:20.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:20.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:20.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:20.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:20.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:20.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:20.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:20.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:20.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:20.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10948], [object_number=79, tag_tid=17, entry_tid=10947], [object_number=78, tag_tid=17, entry_tid=10946], [object_number=77, tag_tid=17, entry_tid=10945]]], state=connected] 2024-09-14T07:01:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:20.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:20.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:20.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:20.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:20.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:20.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=10948], [object_number=79, tag_tid=17, entry_tid=10947], [object_number=78, tag_tid=17, entry_tid=10946], [object_number=77, tag_tid=17, entry_tid=10945]]], state=connected] 2024-09-14T07:01:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10948' '!=' 'object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:20.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:20.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=10948' = 'object_number=79, tag_tid=17, entry_tid=10843' 2024-09-14T07:01:20.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:20.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:20.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:20.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:20.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:20.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:20.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:20.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10948 2024-09-14T07:01:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 10948 2024-09-14T07:01:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:20.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:20.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:20.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:20.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11045], [object_number=76, tag_tid=17, entry_tid=11044], [object_number=79, tag_tid=17, entry_tid=11043], [object_number=78, tag_tid=17, entry_tid=11042]]], state=connected] 2024-09-14T07:01:20.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11045], [object_number=76, tag_tid=17, entry_tid=11044], [object_number=79, tag_tid=17, entry_tid=11043], [object_number=78, tag_tid=17, entry_tid=11042]]], state=connected] 2024-09-14T07:01:20.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11045' '!=' 'object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11045' = 'object_number=76, tag_tid=17, entry_tid=10948' 2024-09-14T07:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:20.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:20.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:20.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:20.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11045 2024-09-14T07:01:20.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11045 2024-09-14T07:01:20.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:20.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:20.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:21.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:21.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:21.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:21.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:21.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:21.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:21.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:21.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:21.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:21.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:21.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:21.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:21.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:21.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:21.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:21.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:21.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:21.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:21.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:21.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:21.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11137], [object_number=76, tag_tid=17, entry_tid=11136], [object_number=79, tag_tid=17, entry_tid=11135], [object_number=78, tag_tid=17, entry_tid=11134]]], state=connected] 2024-09-14T07:01:21.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:21.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:21.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:21.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:21.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:21.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11137], [object_number=76, tag_tid=17, entry_tid=11136], [object_number=79, tag_tid=17, entry_tid=11135], [object_number=78, tag_tid=17, entry_tid=11134]]], state=connected] 2024-09-14T07:01:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11137' '!=' 'object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11137' = 'object_number=77, tag_tid=17, entry_tid=11045' 2024-09-14T07:01:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:21.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11137 2024-09-14T07:01:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11137 2024-09-14T07:01:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:21.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:21.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:21.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:21.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:21.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:21.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:21.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:21.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:21.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:21.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:21.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:21.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=11227], [object_number=78, tag_tid=17, entry_tid=11226], [object_number=77, tag_tid=17, entry_tid=11225], [object_number=76, tag_tid=17, entry_tid=11224]]], state=connected] 2024-09-14T07:01:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:21.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=11227], [object_number=78, tag_tid=17, entry_tid=11226], [object_number=77, tag_tid=17, entry_tid=11225], [object_number=76, tag_tid=17, entry_tid=11224]]], state=connected] 2024-09-14T07:01:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:21.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=11227' '!=' 'object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:21.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=11227' = 'object_number=77, tag_tid=17, entry_tid=11137' 2024-09-14T07:01:21.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:21.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:21.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:21.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:21.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:21.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:21.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:21.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11227 2024-09-14T07:01:21.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11227 2024-09-14T07:01:21.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:21.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:21.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:22.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:22.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:22.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:22.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:22.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:22.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:22.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:22.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:22.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:22.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:22.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:22.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:22.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:22.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:22.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11324], [object_number=79, tag_tid=17, entry_tid=11323], [object_number=78, tag_tid=17, entry_tid=11322], [object_number=77, tag_tid=17, entry_tid=11321]]], state=connected] 2024-09-14T07:01:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11324], [object_number=79, tag_tid=17, entry_tid=11323], [object_number=78, tag_tid=17, entry_tid=11322], [object_number=77, tag_tid=17, entry_tid=11321]]], state=connected] 2024-09-14T07:01:22.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11324' '!=' 'object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11324' = 'object_number=79, tag_tid=17, entry_tid=11227' 2024-09-14T07:01:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:22.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:22.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:22.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:22.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11324 2024-09-14T07:01:22.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11324 2024-09-14T07:01:22.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:22.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:22.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:22.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:22.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:22.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:22.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:22.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:22.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:22.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:22.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:22.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:22.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:22.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:22.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:22.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:22.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:22.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:22.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:22.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:22.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:22.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:22.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11429], [object_number=76, tag_tid=17, entry_tid=11428], [object_number=79, tag_tid=17, entry_tid=11427], [object_number=78, tag_tid=17, entry_tid=11426]]], state=connected] 2024-09-14T07:01:22.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:22.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:22.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:22.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11429], [object_number=76, tag_tid=17, entry_tid=11428], [object_number=79, tag_tid=17, entry_tid=11427], [object_number=78, tag_tid=17, entry_tid=11426]]], state=connected] 2024-09-14T07:01:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11429' '!=' 'object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11429' = 'object_number=76, tag_tid=17, entry_tid=11324' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:22.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:22.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:22.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11429 2024-09-14T07:01:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11429 2024-09-14T07:01:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:23.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:23.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:23.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:23.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:23.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:23.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:23.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:23.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:23.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:23.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:23.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11544], [object_number=79, tag_tid=17, entry_tid=11543], [object_number=78, tag_tid=17, entry_tid=11542], [object_number=77, tag_tid=17, entry_tid=11541]]], state=connected] 2024-09-14T07:01:23.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:23.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:23.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:23.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11544], [object_number=79, tag_tid=17, entry_tid=11543], [object_number=78, tag_tid=17, entry_tid=11542], [object_number=77, tag_tid=17, entry_tid=11541]]], state=connected] 2024-09-14T07:01:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11544' '!=' 'object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11544' = 'object_number=77, tag_tid=17, entry_tid=11429' 2024-09-14T07:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:23.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11544 2024-09-14T07:01:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11544 2024-09-14T07:01:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:23.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:23.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:23.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:23.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:23.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:23.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:23.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:23.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:23.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:23.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:23.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11660], [object_number=79, tag_tid=17, entry_tid=11659], [object_number=78, tag_tid=17, entry_tid=11658], [object_number=77, tag_tid=17, entry_tid=11657]]], state=connected] 2024-09-14T07:01:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:23.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:23.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:23.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:23.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11660], [object_number=79, tag_tid=17, entry_tid=11659], [object_number=78, tag_tid=17, entry_tid=11658], [object_number=77, tag_tid=17, entry_tid=11657]]], state=connected] 2024-09-14T07:01:23.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:23.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11660' '!=' 'object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11660' = 'object_number=76, tag_tid=17, entry_tid=11544' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:23.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:23.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:23.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:23.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:23.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:23.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:23.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:23.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:23.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:23.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11660 2024-09-14T07:01:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11660 2024-09-14T07:01:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:23.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:24.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:24.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:24.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:24.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:24.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:24.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:24.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:24.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:24.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:24.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:24.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11757], [object_number=76, tag_tid=17, entry_tid=11756], [object_number=79, tag_tid=17, entry_tid=11755], [object_number=78, tag_tid=17, entry_tid=11754]]], state=connected] 2024-09-14T07:01:24.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11757], [object_number=76, tag_tid=17, entry_tid=11756], [object_number=79, tag_tid=17, entry_tid=11755], [object_number=78, tag_tid=17, entry_tid=11754]]], state=connected] 2024-09-14T07:01:24.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11757' '!=' 'object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11757' = 'object_number=76, tag_tid=17, entry_tid=11660' 2024-09-14T07:01:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11757 2024-09-14T07:01:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11757 2024-09-14T07:01:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:24.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:24.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:24.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:24.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:24.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:24.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:24.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:24.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11848], [object_number=79, tag_tid=17, entry_tid=11847], [object_number=78, tag_tid=17, entry_tid=11846], [object_number=77, tag_tid=17, entry_tid=11845]]], state=connected] 2024-09-14T07:01:24.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:24.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:24.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:24.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:24.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=11848], [object_number=79, tag_tid=17, entry_tid=11847], [object_number=78, tag_tid=17, entry_tid=11846], [object_number=77, tag_tid=17, entry_tid=11845]]], state=connected] 2024-09-14T07:01:24.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11848' '!=' 'object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=11848' = 'object_number=77, tag_tid=17, entry_tid=11757' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11848 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11848 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:25.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:25.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:25.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:25.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:25.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:25.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:25.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:25.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:25.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:25.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:25.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:25.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:25.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:25.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:25.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:25.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:25.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:25.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:25.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:25.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:25.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:25.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:25.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:25.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:25.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:25.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:25.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11933], [object_number=76, tag_tid=17, entry_tid=11932], [object_number=79, tag_tid=17, entry_tid=11931], [object_number=78, tag_tid=17, entry_tid=11930]]], state=connected] 2024-09-14T07:01:25.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:25.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:25.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:25.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:25.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:25.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:25.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=11933], [object_number=76, tag_tid=17, entry_tid=11932], [object_number=79, tag_tid=17, entry_tid=11931], [object_number=78, tag_tid=17, entry_tid=11930]]], state=connected] 2024-09-14T07:01:25.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11933' '!=' 'object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=11933' = 'object_number=76, tag_tid=17, entry_tid=11848' 2024-09-14T07:01:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:25.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:25.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:25.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:25.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11933 2024-09-14T07:01:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 11933 2024-09-14T07:01:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:25.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:25.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:25.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:25.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:25.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:25.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:25.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:25.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:25.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:25.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:25.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=12023], [object_number=78, tag_tid=17, entry_tid=12022], [object_number=77, tag_tid=17, entry_tid=12021], [object_number=76, tag_tid=17, entry_tid=12020]]], state=connected] 2024-09-14T07:01:25.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:25.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:25.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:25.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:25.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:25.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=12023], [object_number=78, tag_tid=17, entry_tid=12022], [object_number=77, tag_tid=17, entry_tid=12021], [object_number=76, tag_tid=17, entry_tid=12020]]], state=connected] 2024-09-14T07:01:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=12023' '!=' 'object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=12023' = 'object_number=77, tag_tid=17, entry_tid=11933' 2024-09-14T07:01:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:25.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:25.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:25.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:25.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:25.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12023 2024-09-14T07:01:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12023 2024-09-14T07:01:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:25.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:25.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:26.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:26.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:26.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:26.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:26.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:26.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:26.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:26.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:26.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:26.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:26.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:26.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:26.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:26.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:26.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:26.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=12125], [object_number=76, tag_tid=17, entry_tid=12124], [object_number=79, tag_tid=17, entry_tid=12123], [object_number=78, tag_tid=17, entry_tid=12122]]], state=connected] 2024-09-14T07:01:26.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:26.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:26.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:26.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:26.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=12125], [object_number=76, tag_tid=17, entry_tid=12124], [object_number=79, tag_tid=17, entry_tid=12123], [object_number=78, tag_tid=17, entry_tid=12122]]], state=connected] 2024-09-14T07:01:26.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=12125' '!=' 'object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=12125' = 'object_number=79, tag_tid=17, entry_tid=12023' 2024-09-14T07:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:26.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:26.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:26.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12125 2024-09-14T07:01:26.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12125 2024-09-14T07:01:26.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:26.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:26.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:26.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:26.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:26.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:26.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:26.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:26.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:26.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:26.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:26.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:26.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:26.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:26.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:26.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:26.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:26.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:26.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:26.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:26.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:26.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:26.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:26.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:26.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:26.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=12229], [object_number=76, tag_tid=17, entry_tid=12228], [object_number=79, tag_tid=17, entry_tid=12227], [object_number=78, tag_tid=17, entry_tid=12226]]], state=connected] 2024-09-14T07:01:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:26.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=17, entry_tid=12229], [object_number=76, tag_tid=17, entry_tid=12228], [object_number=79, tag_tid=17, entry_tid=12227], [object_number=78, tag_tid=17, entry_tid=12226]]], state=connected] 2024-09-14T07:01:26.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=12229' '!=' 'object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=77, tag_tid=17, entry_tid=12229' = 'object_number=77, tag_tid=17, entry_tid=12125' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12229 2024-09-14T07:01:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12229 2024-09-14T07:01:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:27.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:27.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:27.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:27.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:27.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:27.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:27.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:27.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:27.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:27.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:27.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:27.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:27.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:27.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=12320], [object_number=79, tag_tid=17, entry_tid=12319], [object_number=78, tag_tid=17, entry_tid=12318], [object_number=77, tag_tid=17, entry_tid=12317]]], state=connected] 2024-09-14T07:01:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:27.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:27.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=12320], [object_number=79, tag_tid=17, entry_tid=12319], [object_number=78, tag_tid=17, entry_tid=12318], [object_number=77, tag_tid=17, entry_tid=12317]]], state=connected] 2024-09-14T07:01:27.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=12320' '!=' 'object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=12320' = 'object_number=77, tag_tid=17, entry_tid=12229' 2024-09-14T07:01:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12320 2024-09-14T07:01:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12320 2024-09-14T07:01:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:27.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:27.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:27.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:27.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:27.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:27.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:27.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:27.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:27.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:27.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=12418], [object_number=77, tag_tid=17, entry_tid=12417], [object_number=76, tag_tid=17, entry_tid=12416], [object_number=79, tag_tid=17, entry_tid=12415]]], state=connected] 2024-09-14T07:01:27.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:27.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:27.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:27.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:27.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:27.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:27.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=12418], [object_number=77, tag_tid=17, entry_tid=12417], [object_number=76, tag_tid=17, entry_tid=12416], [object_number=79, tag_tid=17, entry_tid=12415]]], state=connected] 2024-09-14T07:01:27.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=12418' '!=' 'object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=12418' = 'object_number=76, tag_tid=17, entry_tid=12320' 2024-09-14T07:01:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:27.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:27.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:27.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12418 2024-09-14T07:01:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12418 2024-09-14T07:01:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:28.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:28.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:28.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:28.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:28.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:28.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:28.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:28.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:28.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:28.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:28.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:28.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:28.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=12524], [object_number=79, tag_tid=17, entry_tid=12523], [object_number=78, tag_tid=17, entry_tid=12522], [object_number=77, tag_tid=17, entry_tid=12521]]], state=connected] 2024-09-14T07:01:28.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:28.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:28.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=76, tag_tid=17, entry_tid=12524], [object_number=79, tag_tid=17, entry_tid=12523], [object_number=78, tag_tid=17, entry_tid=12522], [object_number=77, tag_tid=17, entry_tid=12521]]], state=connected] 2024-09-14T07:01:28.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=12524' '!=' 'object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=76, tag_tid=17, entry_tid=12524' = 'object_number=78, tag_tid=17, entry_tid=12418' 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12524 2024-09-14T07:01:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12524 2024-09-14T07:01:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:28.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:28.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:28.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:28.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:28.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:28.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:28.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:28.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:28.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:28.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:28.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:28.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:28.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=12626], [object_number=77, tag_tid=17, entry_tid=12625], [object_number=76, tag_tid=17, entry_tid=12624], [object_number=79, tag_tid=17, entry_tid=12623]]], state=connected] 2024-09-14T07:01:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:28.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:28.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=78, tag_tid=17, entry_tid=12626], [object_number=77, tag_tid=17, entry_tid=12625], [object_number=76, tag_tid=17, entry_tid=12624], [object_number=79, tag_tid=17, entry_tid=12623]]], state=connected] 2024-09-14T07:01:28.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:28.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=12626' '!=' 'object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=78, tag_tid=17, entry_tid=12626' = 'object_number=76, tag_tid=17, entry_tid=12524' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:28.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:28.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:28.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:28.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:28.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:28.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:28.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:28.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:28.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:28.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12626 2024-09-14T07:01:28.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12626 2024-09-14T07:01:28.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:28.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:28.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:29.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:29.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:29.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:29.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:29.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:29.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:29.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:29.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:29.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:29.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:29.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=12719], [object_number=78, tag_tid=17, entry_tid=12718], [object_number=77, tag_tid=17, entry_tid=12717], [object_number=76, tag_tid=17, entry_tid=12716]]], state=connected] 2024-09-14T07:01:29.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:29.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:29.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:29.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=79, tag_tid=17, entry_tid=12719], [object_number=78, tag_tid=17, entry_tid=12718], [object_number=77, tag_tid=17, entry_tid=12717], [object_number=76, tag_tid=17, entry_tid=12716]]], state=connected] 2024-09-14T07:01:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=79, tag_tid=17, entry_tid=12719' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=79, tag_tid=17, entry_tid=12719' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=12719' '!=' 'object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=79, tag_tid=17, entry_tid=12719' = 'object_number=78, tag_tid=17, entry_tid=12626' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=79, tag_tid=17, entry_tid=12719' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=18 2024-09-14T07:01:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=12719' 2024-09-14T07:01:29.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:01:29.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:29.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=17 2024-09-14T07:01:29.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:29.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:29.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:29.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:01:29.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=79, tag_tid=17, entry_tid=12719' 2024-09-14T07:01:29.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:01:29.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:01:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12719 2024-09-14T07:01:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 18 = 17 -a 1 -le 12719 2024-09-14T07:01:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:01:29.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:01:29.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:01:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:01:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:01:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:01:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:01:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:01:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:01:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:01:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:01:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:01:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:01:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:01:29.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:01:29.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:01:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:01:29.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:01:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:29.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:29.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:29.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:29.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:29.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:29.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:29.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:29.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:01:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:01:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:01:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:01:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 19 2024-09-14T07:01:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 19 2024-09-14T07:01:29.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:01:29.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:29.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=77, tag_tid=18, entry_tid=1], [object_number=76, tag_tid=18, entry_tid=0], [object_number=79, tag_tid=17, entry_tid=12735], [object_number=78, tag_tid=17, entry_tid=12734]]], state=connected] 2024-09-14T07:01:29.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=77, tag_tid=18, entry_tid=1' -a 'object_number=77, tag_tid=18, entry_tid=1' = 'object_number=77, tag_tid=18, entry_tid=1' 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:29.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6 2024-09-14T07:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:01:29.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:01:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2024-09-14T07:01:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:01:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:01:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:01:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 11 2024-09-14T07:01:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:00:17 2024 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:01:01 2024 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:01:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:01:30.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap5-clean ']' 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap5-clean 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5-clean 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:01:31.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:01:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:32.573 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:01:32.573+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:01:32.576 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:01:32.577+0000 7fea5139f640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:01:32.724 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5129519226556507) > weight (0.25) 2024-09-14T07:01:32.725 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T07:01:32.725 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T07:01:32.725 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T07:01:32.725 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T07:01:32.725 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:01:32.726 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:01:32.724+0000 7f4c9736d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 45817) UID: 1000 2024-09-14T07:01:32.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:01:32.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap6 2024-09-14T07:01:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:33.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6 2024-09-14T07:01:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:01:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:01:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:01:33.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:01:33.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:34.679 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:01:34.681+0000 7f99d920e640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:01:34.679 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:01:34.681+0000 7f99dda17640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:01:35.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:01:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:35.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:01:35.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:35.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap6-clean 2024-09-14T07:01:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap6-clean 2024-09-14T07:01:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:01:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6-clean 2024-09-14T07:01:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T07:01:36.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:01:36.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T07:01:36.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:01:36.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:36.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:36.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:01:36.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:01:36.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:01:36.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T07:01:38.827 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T07:01:38.827 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T07:01:38.828 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T07:01:38.830 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T07:01:38.830 INFO:tasks.rbd_mirror_thrash:waiting for 66 secs before thrashing 2024-09-14T07:01:40.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 90018816 bytes 2024-09-14T07:01:44.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 150723584 bytes 2024-09-14T07:01:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:01:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:01:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:01:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:01:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:01:45.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T07:01:45.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":2734283.3,"entries_behind_primary":0,"entries_per_second":170.77,"non_primary_position":{"entry_tid":1,"object_number":77,"tag_tid":18},"primary_position":{"entry_tid":1,"object_number":77,"tag_tid":18}} 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:01:40 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:01:40' 2024-09-14T07:01:45.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T07:01:45.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T07:01:45.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T07:01:45.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":2734283.3,"entries_behind_primary":0,"entries_per_second":170.77,"non_primary_position":{"entry_tid":1,"object_number":77,"tag_tid":18},"primary_position":{"entry_tid":1,"object_number":77,"tag_tid":18}} 2024-09-14T07:01:45.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:01:45.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:01:45.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T07:01:45.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:01:45.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T07:01:45.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T07:01:45.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T07:01:45.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T07:01:45.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:01:40' 2024-09-14T07:01:45.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 07:01:40' 2024-09-14T07:01:45.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:01:45.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:01:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 07:01:40' 2024-09-14T07:01:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:01:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:01:45.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T07:01:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK43094312mirror.3smithi16919.3.0-4775-g7f24e589trueOK44074410mirror.2smithi16919.3.0-4775-g7f24e589falseOK44494452mirror.1smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":2734283.3,"entries_behind_primary":0,"entries_per_second":170.77,"non_primary_position":{"entry_tid":1,"object_number":77,"tag_tid":18},"primary_position":{"entry_tid":1,"object_number":77,"tag_tid":18}}42104261mirror.0smithi1692024-09-14 07:01:40cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 07:01:40' 2024-09-14T07:01:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T07:01:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T07:01:45.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:01:40' 2024-09-14T07:01:45.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T07:01:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T07:01:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:01:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T07:01:45.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap7 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap7 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7 2024-09-14T07:01:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:02:16.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T07:02:16.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:02:17.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:17.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:17.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:17.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:02:17.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:02:17.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:02:17.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:17.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:02:17.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:02:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:17.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:17.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:17.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:17.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:02:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:02:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:02:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:02:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:02:17.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:02:17.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:17.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:17.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:17.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:17.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:17.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:17.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:17.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:17.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:17.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:17.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:17.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:17.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:17.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:17.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:17.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:17.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:17.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:17.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:17.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:17.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:17.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=7008], [object_number=87, tag_tid=20, entry_tid=7007], [object_number=86, tag_tid=20, entry_tid=7006], [object_number=85, tag_tid=20, entry_tid=7005]]], state=connected] 2024-09-14T07:02:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:17.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=7008], [object_number=87, tag_tid=20, entry_tid=7007], [object_number=86, tag_tid=20, entry_tid=7006], [object_number=85, tag_tid=20, entry_tid=7005]]], state=connected] 2024-09-14T07:02:17.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:17.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:17.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:17.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=7008' '!=' '' 2024-09-14T07:02:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=7008' = '' 2024-09-14T07:02:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:17.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:17.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:17.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:17.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:17.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:17.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:17.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:17.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:17.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:17.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:17.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7008 2024-09-14T07:02:17.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7008 2024-09-14T07:02:17.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:17.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:17.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:18.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:18.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:18.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:18.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:18.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:18.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:18.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:18.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:18.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:18.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:18.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:18.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:18.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:18.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:18.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=7102], [object_number=85, tag_tid=20, entry_tid=7101], [object_number=84, tag_tid=20, entry_tid=7100], [object_number=87, tag_tid=20, entry_tid=7099]]], state=connected] 2024-09-14T07:02:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:18.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:18.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:18.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:18.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:18.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:18.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:18.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:18.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:18.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=7102], [object_number=85, tag_tid=20, entry_tid=7101], [object_number=84, tag_tid=20, entry_tid=7100], [object_number=87, tag_tid=20, entry_tid=7099]]], state=connected] 2024-09-14T07:02:18.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=7102' '!=' 'object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=7102' = 'object_number=84, tag_tid=20, entry_tid=7008' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:18.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:18.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:18.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:18.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:18.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:18.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:18.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:18.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7102 2024-09-14T07:02:18.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7102 2024-09-14T07:02:18.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:18.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:18.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:18.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:18.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:18.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:18.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:18.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:18.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:18.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:18.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:18.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:18.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:18.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:18.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:18.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:18.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:18.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:18.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:18.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=7197], [object_number=84, tag_tid=20, entry_tid=7196], [object_number=87, tag_tid=20, entry_tid=7195], [object_number=86, tag_tid=20, entry_tid=7194]]], state=connected] 2024-09-14T07:02:18.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:18.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:18.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:18.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:18.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=7197], [object_number=84, tag_tid=20, entry_tid=7196], [object_number=87, tag_tid=20, entry_tid=7195], [object_number=86, tag_tid=20, entry_tid=7194]]], state=connected] 2024-09-14T07:02:18.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=7197' '!=' 'object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=7197' = 'object_number=86, tag_tid=20, entry_tid=7102' 2024-09-14T07:02:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:18.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:18.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:18.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:18.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:18.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:18.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:18.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:18.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:18.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:18.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7197 2024-09-14T07:02:18.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7197 2024-09-14T07:02:18.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:18.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:18.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:19.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:19.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:19.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:19.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:19.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:19.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:19.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:19.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:19.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:19.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=7302], [object_number=85, tag_tid=20, entry_tid=7301], [object_number=84, tag_tid=20, entry_tid=7300], [object_number=87, tag_tid=20, entry_tid=7299]]], state=connected] 2024-09-14T07:02:19.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:19.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:19.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:19.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:19.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=7302], [object_number=85, tag_tid=20, entry_tid=7301], [object_number=84, tag_tid=20, entry_tid=7300], [object_number=87, tag_tid=20, entry_tid=7299]]], state=connected] 2024-09-14T07:02:19.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=7302' '!=' 'object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:19.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:19.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=7302' = 'object_number=85, tag_tid=20, entry_tid=7197' 2024-09-14T07:02:19.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:19.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:19.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:19.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:19.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7302 2024-09-14T07:02:19.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7302 2024-09-14T07:02:19.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:19.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:19.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:19.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:19.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:19.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:19.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:19.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:19.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:19.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:19.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:19.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:19.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:19.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:19.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:19.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:19.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:19.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:19.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:19.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:19.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:19.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=7412], [object_number=87, tag_tid=20, entry_tid=7411], [object_number=86, tag_tid=20, entry_tid=7410], [object_number=85, tag_tid=20, entry_tid=7409]]], state=connected] 2024-09-14T07:02:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:19.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:19.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:19.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:19.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=7412], [object_number=87, tag_tid=20, entry_tid=7411], [object_number=86, tag_tid=20, entry_tid=7410], [object_number=85, tag_tid=20, entry_tid=7409]]], state=connected] 2024-09-14T07:02:19.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=7412' '!=' 'object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=7412' = 'object_number=86, tag_tid=20, entry_tid=7302' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:19.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:19.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:19.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:19.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:19.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:19.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:19.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:19.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:19.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:19.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:19.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7412 2024-09-14T07:02:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7412 2024-09-14T07:02:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:20.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:20.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:20.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:20.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:20.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:20.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=7502], [object_number=85, tag_tid=20, entry_tid=7501], [object_number=84, tag_tid=20, entry_tid=7500], [object_number=87, tag_tid=20, entry_tid=7499]]], state=connected] 2024-09-14T07:02:20.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:20.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:20.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:20.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=7502], [object_number=85, tag_tid=20, entry_tid=7501], [object_number=84, tag_tid=20, entry_tid=7500], [object_number=87, tag_tid=20, entry_tid=7499]]], state=connected] 2024-09-14T07:02:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=7502' '!=' 'object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:20.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=7502' = 'object_number=84, tag_tid=20, entry_tid=7412' 2024-09-14T07:02:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:20.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:20.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:20.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:20.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:20.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7502 2024-09-14T07:02:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7502 2024-09-14T07:02:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:20.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:20.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:20.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:20.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:20.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:20.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:20.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:20.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:20.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=7600], [object_number=87, tag_tid=20, entry_tid=7599], [object_number=86, tag_tid=20, entry_tid=7598], [object_number=85, tag_tid=20, entry_tid=7597]]], state=connected] 2024-09-14T07:02:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:21.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:21.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:21.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:21.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:21.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:21.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=7600], [object_number=87, tag_tid=20, entry_tid=7599], [object_number=86, tag_tid=20, entry_tid=7598], [object_number=85, tag_tid=20, entry_tid=7597]]], state=connected] 2024-09-14T07:02:21.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=7600' '!=' 'object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=7600' = 'object_number=86, tag_tid=20, entry_tid=7502' 2024-09-14T07:02:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:21.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:21.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:21.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:21.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:21.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:21.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:21.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:21.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:21.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:21.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:21.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:21.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:21.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:21.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:21.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7600 2024-09-14T07:02:21.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7600 2024-09-14T07:02:21.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:21.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:21.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:21.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:21.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:21.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:21.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:21.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:21.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:21.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:21.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:21.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:21.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:21.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:21.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:21.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:21.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:21.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:21.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:21.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:21.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:21.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=87, tag_tid=20, entry_tid=7779], [object_number=86, tag_tid=20, entry_tid=7778], [object_number=85, tag_tid=20, entry_tid=7777], [object_number=84, tag_tid=20, entry_tid=7776]]], state=connected] 2024-09-14T07:02:21.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:21.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:21.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:21.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:21.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:21.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:21.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:21.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=87, tag_tid=20, entry_tid=7779], [object_number=86, tag_tid=20, entry_tid=7778], [object_number=85, tag_tid=20, entry_tid=7777], [object_number=84, tag_tid=20, entry_tid=7776]]], state=connected] 2024-09-14T07:02:22.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=87, tag_tid=20, entry_tid=7779' '!=' 'object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=87, tag_tid=20, entry_tid=7779' = 'object_number=84, tag_tid=20, entry_tid=7600' 2024-09-14T07:02:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:22.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7779 2024-09-14T07:02:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7779 2024-09-14T07:02:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:22.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:22.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:22.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:22.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:22.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:22.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:22.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:22.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:22.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:22.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:22.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:22.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:22.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:22.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:22.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:22.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:22.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:22.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:22.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=7913], [object_number=84, tag_tid=20, entry_tid=7912], [object_number=87, tag_tid=20, entry_tid=7911], [object_number=86, tag_tid=20, entry_tid=7910]]], state=connected] 2024-09-14T07:02:22.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:22.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:22.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:22.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:22.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:22.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=7913], [object_number=84, tag_tid=20, entry_tid=7912], [object_number=87, tag_tid=20, entry_tid=7911], [object_number=86, tag_tid=20, entry_tid=7910]]], state=connected] 2024-09-14T07:02:22.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=7913' '!=' 'object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=7913' = 'object_number=87, tag_tid=20, entry_tid=7779' 2024-09-14T07:02:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:22.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:22.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:22.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:22.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:22.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:22.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:22.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:22.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7913 2024-09-14T07:02:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 7913 2024-09-14T07:02:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:22.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:22.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:22.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:22.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:22.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:22.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:22.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:22.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:22.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:22.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:22.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:22.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:22.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:22.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=8000], [object_number=87, tag_tid=20, entry_tid=7999], [object_number=86, tag_tid=20, entry_tid=7998], [object_number=85, tag_tid=20, entry_tid=7997]]], state=connected] 2024-09-14T07:02:22.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:22.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:22.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:22.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:22.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:23.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=8000], [object_number=87, tag_tid=20, entry_tid=7999], [object_number=86, tag_tid=20, entry_tid=7998], [object_number=85, tag_tid=20, entry_tid=7997]]], state=connected] 2024-09-14T07:02:23.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=8000' '!=' 'object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:23.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:23.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=8000' = 'object_number=85, tag_tid=20, entry_tid=7913' 2024-09-14T07:02:23.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:23.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:23.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:23.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:23.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:23.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:23.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:23.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8000 2024-09-14T07:02:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8000 2024-09-14T07:02:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:23.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:23.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:23.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:23.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:23.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:23.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:23.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:23.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=87, tag_tid=20, entry_tid=8143], [object_number=86, tag_tid=20, entry_tid=8142], [object_number=85, tag_tid=20, entry_tid=8141], [object_number=84, tag_tid=20, entry_tid=8140]]], state=connected] 2024-09-14T07:02:23.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:23.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:23.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:23.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:23.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:23.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:23.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=87, tag_tid=20, entry_tid=8143], [object_number=86, tag_tid=20, entry_tid=8142], [object_number=85, tag_tid=20, entry_tid=8141], [object_number=84, tag_tid=20, entry_tid=8140]]], state=connected] 2024-09-14T07:02:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=87, tag_tid=20, entry_tid=8143' '!=' 'object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=87, tag_tid=20, entry_tid=8143' = 'object_number=84, tag_tid=20, entry_tid=8000' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:23.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8143 2024-09-14T07:02:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8143 2024-09-14T07:02:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:23.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:23.729 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.35903960766869814) > weight (0.25) 2024-09-14T07:02:23.729 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.36627503334197986) > weight (0.25) 2024-09-14T07:02:23.729 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8756758517713155) > weight (0.25) 2024-09-14T07:02:23.729 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-14T07:02:23.729 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-14T07:02:23.730 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T07:02:23.730 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:02:23.731+0000 7f64593e5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 31832) UID: 1000 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:23.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:23.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:23.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:23.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:23.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:23.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:23.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:23.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:23.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:24.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:24.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:24.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:24.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:24.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:24.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:24.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=8249], [object_number=84, tag_tid=20, entry_tid=8248], [object_number=87, tag_tid=20, entry_tid=8247], [object_number=86, tag_tid=20, entry_tid=8246]]], state=connected] 2024-09-14T07:02:24.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=8249], [object_number=84, tag_tid=20, entry_tid=8248], [object_number=87, tag_tid=20, entry_tid=8247], [object_number=86, tag_tid=20, entry_tid=8246]]], state=connected] 2024-09-14T07:02:24.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=8249' '!=' 'object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=8249' = 'object_number=87, tag_tid=20, entry_tid=8143' 2024-09-14T07:02:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:24.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:24.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:24.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:24.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:24.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:24.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:24.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:24.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:24.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8249 2024-09-14T07:02:24.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8249 2024-09-14T07:02:24.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:24.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:24.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:24.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:24.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:24.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:24.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:24.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:24.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:24.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:24.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:24.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:24.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:24.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:24.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:24.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:24.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=87, tag_tid=20, entry_tid=8355], [object_number=86, tag_tid=20, entry_tid=8354], [object_number=85, tag_tid=20, entry_tid=8353], [object_number=84, tag_tid=20, entry_tid=8352]]], state=connected] 2024-09-14T07:02:24.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:24.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:24.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:24.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:24.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=87, tag_tid=20, entry_tid=8355], [object_number=86, tag_tid=20, entry_tid=8354], [object_number=85, tag_tid=20, entry_tid=8353], [object_number=84, tag_tid=20, entry_tid=8352]]], state=connected] 2024-09-14T07:02:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=87, tag_tid=20, entry_tid=8355' '!=' 'object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=87, tag_tid=20, entry_tid=8355' = 'object_number=85, tag_tid=20, entry_tid=8249' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:24.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:24.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8355 2024-09-14T07:02:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8355 2024-09-14T07:02:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:24.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:24.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:24.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:24.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:25.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:25.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:25.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:25.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=8408], [object_number=87, tag_tid=20, entry_tid=8407], [object_number=86, tag_tid=20, entry_tid=8406], [object_number=85, tag_tid=20, entry_tid=8405]]], state=connected] 2024-09-14T07:02:25.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:25.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=8408], [object_number=87, tag_tid=20, entry_tid=8407], [object_number=86, tag_tid=20, entry_tid=8406], [object_number=85, tag_tid=20, entry_tid=8405]]], state=connected] 2024-09-14T07:02:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=8408' '!=' 'object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=8408' = 'object_number=87, tag_tid=20, entry_tid=8355' 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8408 2024-09-14T07:02:25.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8408 2024-09-14T07:02:25.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:25.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:25.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:25.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:25.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:25.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:25.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:25.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:25.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=8482], [object_number=85, tag_tid=20, entry_tid=8481], [object_number=84, tag_tid=20, entry_tid=8480], [object_number=87, tag_tid=20, entry_tid=8479]]], state=connected] 2024-09-14T07:02:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=8482], [object_number=85, tag_tid=20, entry_tid=8481], [object_number=84, tag_tid=20, entry_tid=8480], [object_number=87, tag_tid=20, entry_tid=8479]]], state=connected] 2024-09-14T07:02:25.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=8482' '!=' 'object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=8482' = 'object_number=84, tag_tid=20, entry_tid=8408' 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:25.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:25.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:25.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:25.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8482 2024-09-14T07:02:25.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8482 2024-09-14T07:02:25.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:25.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:25.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:25.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:25.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:25.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:25.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:25.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:26.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:26.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:26.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:26.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:26.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:26.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=8578], [object_number=85, tag_tid=20, entry_tid=8577], [object_number=84, tag_tid=20, entry_tid=8576], [object_number=87, tag_tid=20, entry_tid=8575]]], state=connected] 2024-09-14T07:02:26.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=8578], [object_number=85, tag_tid=20, entry_tid=8577], [object_number=84, tag_tid=20, entry_tid=8576], [object_number=87, tag_tid=20, entry_tid=8575]]], state=connected] 2024-09-14T07:02:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=8578' '!=' 'object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=8578' = 'object_number=86, tag_tid=20, entry_tid=8482' 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:26.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:26.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:26.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:26.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:26.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:26.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:26.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8578 2024-09-14T07:02:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8578 2024-09-14T07:02:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:26.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:26.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:26.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:26.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:26.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:26.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:26.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:26.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:26.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:26.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:26.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:26.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:26.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:26.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:26.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:26.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:26.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:26.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:26.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=8678], [object_number=85, tag_tid=20, entry_tid=8677], [object_number=84, tag_tid=20, entry_tid=8676], [object_number=87, tag_tid=20, entry_tid=8675]]], state=connected] 2024-09-14T07:02:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:26.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:26.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:26.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=8678], [object_number=85, tag_tid=20, entry_tid=8677], [object_number=84, tag_tid=20, entry_tid=8676], [object_number=87, tag_tid=20, entry_tid=8675]]], state=connected] 2024-09-14T07:02:26.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=8678' '!=' 'object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=8678' = 'object_number=86, tag_tid=20, entry_tid=8578' 2024-09-14T07:02:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:26.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:26.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:26.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:26.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:26.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:26.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8678 2024-09-14T07:02:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8678 2024-09-14T07:02:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:27.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:27.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:27.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:27.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:27.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:27.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:27.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:27.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:27.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:27.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:27.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:27.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:27.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=8829], [object_number=84, tag_tid=20, entry_tid=8828], [object_number=87, tag_tid=20, entry_tid=8827], [object_number=86, tag_tid=20, entry_tid=8826]]], state=connected] 2024-09-14T07:02:27.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:27.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=85, tag_tid=20, entry_tid=8829], [object_number=84, tag_tid=20, entry_tid=8828], [object_number=87, tag_tid=20, entry_tid=8827], [object_number=86, tag_tid=20, entry_tid=8826]]], state=connected] 2024-09-14T07:02:27.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=8829' '!=' 'object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=85, tag_tid=20, entry_tid=8829' = 'object_number=86, tag_tid=20, entry_tid=8678' 2024-09-14T07:02:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:27.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8829 2024-09-14T07:02:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8829 2024-09-14T07:02:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:27.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:27.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:27.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:27.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:27.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=8936], [object_number=87, tag_tid=20, entry_tid=8935], [object_number=86, tag_tid=20, entry_tid=8934], [object_number=85, tag_tid=20, entry_tid=8933]]], state=connected] 2024-09-14T07:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=84, tag_tid=20, entry_tid=8936], [object_number=87, tag_tid=20, entry_tid=8935], [object_number=86, tag_tid=20, entry_tid=8934], [object_number=85, tag_tid=20, entry_tid=8933]]], state=connected] 2024-09-14T07:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=8936' '!=' 'object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=84, tag_tid=20, entry_tid=8936' = 'object_number=85, tag_tid=20, entry_tid=8829' 2024-09-14T07:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:27.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:27.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:27.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:27.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:27.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:27.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:27.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:27.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8936 2024-09-14T07:02:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 8936 2024-09-14T07:02:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:27.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:28.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:28.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:28.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:28.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:28.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:28.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:28.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:28.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:28.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:28.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:28.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:28.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:28.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=9054], [object_number=85, tag_tid=20, entry_tid=9053], [object_number=84, tag_tid=20, entry_tid=9052], [object_number=87, tag_tid=20, entry_tid=9051]]], state=connected] 2024-09-14T07:02:28.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:28.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 21 2024-09-14T07:02:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:28.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=86, tag_tid=20, entry_tid=9054], [object_number=85, tag_tid=20, entry_tid=9053], [object_number=84, tag_tid=20, entry_tid=9052], [object_number=87, tag_tid=20, entry_tid=9051]]], state=connected] 2024-09-14T07:02:28.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=9054' '!=' 'object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=86, tag_tid=20, entry_tid=9054' = 'object_number=84, tag_tid=20, entry_tid=8936' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:28.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:28.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:28.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:28.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:28.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:28.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:28.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:28.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:28.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:28.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9054 2024-09-14T07:02:28.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9054 2024-09-14T07:02:28.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:28.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:28.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:28.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:28.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:28.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:28.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:28.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:28.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:28.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:28.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:28.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:28.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:28.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:28.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:28.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9149], [object_number=88, tag_tid=20, entry_tid=9148], [object_number=91, tag_tid=20, entry_tid=9147], [object_number=90, tag_tid=20, entry_tid=9146]]], state=connected] 2024-09-14T07:02:28.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:28.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9149], [object_number=88, tag_tid=20, entry_tid=9148], [object_number=91, tag_tid=20, entry_tid=9147], [object_number=90, tag_tid=20, entry_tid=9146]]], state=connected] 2024-09-14T07:02:29.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9149' '!=' 'object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9149' = 'object_number=86, tag_tid=20, entry_tid=9054' 2024-09-14T07:02:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:29.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:29.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:29.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:29.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:29.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9149 2024-09-14T07:02:29.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9149 2024-09-14T07:02:29.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:29.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:29.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:29.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:29.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:29.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:29.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:29.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:29.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9297], [object_number=88, tag_tid=20, entry_tid=9296], [object_number=91, tag_tid=20, entry_tid=9295], [object_number=90, tag_tid=20, entry_tid=9294]]], state=connected] 2024-09-14T07:02:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:29.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9297], [object_number=88, tag_tid=20, entry_tid=9296], [object_number=91, tag_tid=20, entry_tid=9295], [object_number=90, tag_tid=20, entry_tid=9294]]], state=connected] 2024-09-14T07:02:29.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9297' '!=' 'object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9297' = 'object_number=89, tag_tid=20, entry_tid=9149' 2024-09-14T07:02:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:29.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:29.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:29.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9297 2024-09-14T07:02:29.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9297 2024-09-14T07:02:29.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:29.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:29.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:29.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:29.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:29.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:29.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:29.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:29.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:29.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:29.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:29.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:29.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:29.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=88, tag_tid=20, entry_tid=9392], [object_number=91, tag_tid=20, entry_tid=9391], [object_number=90, tag_tid=20, entry_tid=9390], [object_number=89, tag_tid=20, entry_tid=9389]]], state=connected] 2024-09-14T07:02:29.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:30.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:30.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:30.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:30.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=88, tag_tid=20, entry_tid=9392], [object_number=91, tag_tid=20, entry_tid=9391], [object_number=90, tag_tid=20, entry_tid=9390], [object_number=89, tag_tid=20, entry_tid=9389]]], state=connected] 2024-09-14T07:02:30.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=88, tag_tid=20, entry_tid=9392' '!=' 'object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=88, tag_tid=20, entry_tid=9392' = 'object_number=89, tag_tid=20, entry_tid=9297' 2024-09-14T07:02:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9392 2024-09-14T07:02:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9392 2024-09-14T07:02:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:30.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:30.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:30.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:30.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:30.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:30.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:30.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:30.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:30.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9501], [object_number=88, tag_tid=20, entry_tid=9500], [object_number=91, tag_tid=20, entry_tid=9499], [object_number=90, tag_tid=20, entry_tid=9498]]], state=connected] 2024-09-14T07:02:30.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:30.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:30.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9501], [object_number=88, tag_tid=20, entry_tid=9500], [object_number=91, tag_tid=20, entry_tid=9499], [object_number=90, tag_tid=20, entry_tid=9498]]], state=connected] 2024-09-14T07:02:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:30.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9501' '!=' 'object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9501' = 'object_number=88, tag_tid=20, entry_tid=9392' 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:30.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:30.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:30.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:30.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:30.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:30.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9501 2024-09-14T07:02:30.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9501 2024-09-14T07:02:30.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:30.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:30.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:30.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:30.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:30.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:30.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:30.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:30.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:30.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:30.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:30.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:30.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:30.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:30.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:30.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:30.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=91, tag_tid=20, entry_tid=9611], [object_number=90, tag_tid=20, entry_tid=9610], [object_number=89, tag_tid=20, entry_tid=9609], [object_number=88, tag_tid=20, entry_tid=9608]]], state=connected] 2024-09-14T07:02:30.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:30.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:30.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:30.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:31.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:31.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:31.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:31.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=91, tag_tid=20, entry_tid=9611], [object_number=90, tag_tid=20, entry_tid=9610], [object_number=89, tag_tid=20, entry_tid=9609], [object_number=88, tag_tid=20, entry_tid=9608]]], state=connected] 2024-09-14T07:02:31.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=91, tag_tid=20, entry_tid=9611' '!=' 'object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=91, tag_tid=20, entry_tid=9611' = 'object_number=89, tag_tid=20, entry_tid=9501' 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:31.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:31.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:31.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:31.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:31.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:31.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9611 2024-09-14T07:02:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9611 2024-09-14T07:02:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:31.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:31.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:31.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:31.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:31.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:31.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:31.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:31.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9725], [object_number=88, tag_tid=20, entry_tid=9724], [object_number=91, tag_tid=20, entry_tid=9723], [object_number=90, tag_tid=20, entry_tid=9722]]], state=connected] 2024-09-14T07:02:31.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:31.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9725], [object_number=88, tag_tid=20, entry_tid=9724], [object_number=91, tag_tid=20, entry_tid=9723], [object_number=90, tag_tid=20, entry_tid=9722]]], state=connected] 2024-09-14T07:02:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9725' '!=' 'object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9725' = 'object_number=91, tag_tid=20, entry_tid=9611' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:31.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:31.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:31.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:31.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:31.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:31.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:31.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:31.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:31.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9725 2024-09-14T07:02:31.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9725 2024-09-14T07:02:31.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:31.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:31.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:31.730 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-14T07:02:31.731 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:02:31.731 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-14T07:02:31.731 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-14T07:02:31.731 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-14T07:02:31.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:31.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:31.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:31.735 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-14T07:02:31.735 INFO:tasks.rbd_mirror_thrash:waiting for 84 secs before thrashing 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:31.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:31.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:31.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:31.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:31.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:31.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:32.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:32.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=90, tag_tid=20, entry_tid=9830], [object_number=89, tag_tid=20, entry_tid=9829], [object_number=88, tag_tid=20, entry_tid=9828], [object_number=91, tag_tid=20, entry_tid=9827]]], state=connected] 2024-09-14T07:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:32.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:32.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:32.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:32.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:32.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:32.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=90, tag_tid=20, entry_tid=9830], [object_number=89, tag_tid=20, entry_tid=9829], [object_number=88, tag_tid=20, entry_tid=9828], [object_number=91, tag_tid=20, entry_tid=9827]]], state=connected] 2024-09-14T07:02:32.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=90, tag_tid=20, entry_tid=9830' '!=' 'object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:32.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:32.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=90, tag_tid=20, entry_tid=9830' = 'object_number=89, tag_tid=20, entry_tid=9725' 2024-09-14T07:02:32.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:32.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:32.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9830 2024-09-14T07:02:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9830 2024-09-14T07:02:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:32.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:32.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:32.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:32.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:32.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:32.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:32.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:32.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:32.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:32.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:32.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:32.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:32.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:32.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:32.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:32.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9997], [object_number=88, tag_tid=20, entry_tid=9996], [object_number=91, tag_tid=20, entry_tid=9995], [object_number=90, tag_tid=20, entry_tid=9994]]], state=connected] 2024-09-14T07:02:32.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:32.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:32.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:32.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:32.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=20, entry_tid=9997], [object_number=88, tag_tid=20, entry_tid=9996], [object_number=91, tag_tid=20, entry_tid=9995], [object_number=90, tag_tid=20, entry_tid=9994]]], state=connected] 2024-09-14T07:02:32.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9997' '!=' 'object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=89, tag_tid=20, entry_tid=9997' = 'object_number=90, tag_tid=20, entry_tid=9830' 2024-09-14T07:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:32.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9997 2024-09-14T07:02:32.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 9997 2024-09-14T07:02:32.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:32.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:32.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:33.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:33.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:33.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:33.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:33.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:33.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:33.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:33.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:33.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:33.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:33.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:33.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:33.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:33.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030], [object_number=89, tag_tid=20, entry_tid=10029], [object_number=88, tag_tid=20, entry_tid=10028]]], state=connected] 2024-09-14T07:02:33.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030], [object_number=89, tag_tid=20, entry_tid=10029], [object_number=88, tag_tid=20, entry_tid=10028]]], state=connected] 2024-09-14T07:02:33.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=91, tag_tid=20, entry_tid=10031' 2024-09-14T07:02:33.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=91, tag_tid=20, entry_tid=10031' 2024-09-14T07:02:33.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=91, tag_tid=20, entry_tid=10031' '!=' 'object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:33.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:02:33.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=91, tag_tid=20, entry_tid=10031' = 'object_number=89, tag_tid=20, entry_tid=9997' 2024-09-14T07:02:33.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=91, tag_tid=20, entry_tid=10031' 2024-09-14T07:02:33.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:33.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:33.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=21 2024-09-14T07:02:33.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=91, tag_tid=20, entry_tid=10031' 2024-09-14T07:02:33.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:02:33.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=20 2024-09-14T07:02:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:33.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:02:33.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=91, tag_tid=20, entry_tid=10031' 2024-09-14T07:02:33.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:02:33.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:02:33.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10031 2024-09-14T07:02:33.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 21 = 20 -a 1 -le 10031 2024-09-14T07:02:33.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:02:33.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:02:33.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:02:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:02:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:02:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:02:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:02:33.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:02:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:02:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:02:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:02:33.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:02:33.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:02:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:33.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:33.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:02:33.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:02:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:02:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:33.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:33.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:33.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:02:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:02:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:02:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 22 2024-09-14T07:02:33.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 22 2024-09-14T07:02:33.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:02:33.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:33.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=89, tag_tid=21, entry_tid=1], [object_number=88, tag_tid=21, entry_tid=0], [object_number=91, tag_tid=20, entry_tid=10031], [object_number=90, tag_tid=20, entry_tid=10030]]], state=connected] 2024-09-14T07:02:33.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=89, tag_tid=21, entry_tid=1' -a 'object_number=89, tag_tid=21, entry_tid=1' = 'object_number=89, tag_tid=21, entry_tid=1' 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:02:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:02:34.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 13 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:02:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:01:31 2024 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:02:02 2024 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:02:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:02:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap6-clean ']' 2024-09-14T07:02:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap6-clean 2024-09-14T07:02:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6-clean 2024-09-14T07:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:02:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:02:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:36.540 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:02:36.540+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:02:36.544 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:02:36.544+0000 7fea4bb94640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:02:37.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:02:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:37.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap7 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:37.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:02:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:02:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:38.826 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:02:38.828+0000 7f0de0b81640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:02:39.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:02:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:02:39.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap7-clean 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap7-clean 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7-clean 2024-09-14T07:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T07:02:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:02:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T07:02:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:02:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:02:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T07:02:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:02:40.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T07:02:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 628736 bytes 2024-09-14T07:02:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 1252352 bytes 2024-09-14T07:02:43.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 69153792 bytes 2024-09-14T07:02:44.831 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8345559362877951) > weight (0.25) 2024-09-14T07:02:44.831 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T07:02:44.832 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T07:02:44.832 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T07:02:44.832 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:02:44.831+0000 7f3e9a550640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 65285) UID: 1000 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:02:45.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:02:45.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:02:45.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:02:45.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:02:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:02:46.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:02:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:02:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":2589770.3,"entries_behind_primary":0,"entries_per_second":162.5,"non_primary_position":{"entry_tid":1,"object_number":89,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":89,"tag_tid":21}} 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:02:40 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T07:02:46.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T07:02:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T07:02:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:02:40' 2024-09-14T07:02:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T07:02:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T07:02:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T07:02:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":2589770.3,"entries_behind_primary":0,"entries_per_second":162.5,"non_primary_position":{"entry_tid":1,"object_number":89,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":89,"tag_tid":21}} 2024-09-14T07:02:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:02:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:02:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T07:02:46.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:02:46.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T07:02:46.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T07:02:46.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T07:02:46.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:02:40' 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 07:02:40' 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 07:02:40' 2024-09-14T07:02:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:02:46.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:02:46.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T07:02:46.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK44074410mirror.2smithi16919.3.0-4775-g7f24e589trueOK44494452mirror.1smithi16919.3.0-4775-g7f24e589falseOK44794482mirror.3smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":2589770.3,"entries_behind_primary":0,"entries_per_second":162.5,"non_primary_position":{"entry_tid":1,"object_number":89,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":89,"tag_tid":21}}42104261mirror.0smithi1692024-09-14 07:02:40cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 07:02:40' 2024-09-14T07:02:46.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T07:02:46.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T07:02:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:02:40' 2024-09-14T07:02:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T07:02:46.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T07:02:46.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:02:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T07:02:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:02:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap8 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap8 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8 2024-09-14T07:02:46.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:02:46.832 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T07:02:46.832 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:02:46.833 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T07:02:46.833 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T07:02:46.833 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T07:02:46.836 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T07:02:46.836 INFO:tasks.rbd_mirror_thrash:waiting for 98 secs before thrashing 2024-09-14T07:03:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T07:03:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:03:18.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T07:03:18.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:18.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:18.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:18.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:03:18.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:03:18.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:18.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:03:18.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:03:18.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:03:18.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:03:18.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:03:18.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:03:18.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:18.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:03:18.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:03:18.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:03:18.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:03:18.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:03:18.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:03:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:03:18.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:03:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:03:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:03:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:03:18.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:03:18.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:03:18.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:03:18.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:18.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:03:18.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:03:18.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:03:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:18.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:03:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:03:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:03:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:03:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:03:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:03:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:03:18.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:18.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:18.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:18.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:03:18.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:03:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:03:18.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:18.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:18.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:18.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:18.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:19.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=93, tag_tid=24, entry_tid=1' -a 'object_number=93, tag_tid=24, entry_tid=1' = 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=95, tag_tid=23, entry_tid=5903' '!=' '' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=95, tag_tid=23, entry_tid=5903' = '' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:03:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=24 2024-09-14T07:03:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:03:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:03:19.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=23 2024-09-14T07:03:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:03:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:03:19.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:03:19.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:03:19.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:03:19.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=5903 2024-09-14T07:03:19.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 24 = 23 -a 1 -le 5903 2024-09-14T07:03:19.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:03:19.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:03:19.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:03:19.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:03:19.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:03:19.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:03:19.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:03:19.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:03:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:03:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:03:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:03:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:03:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:03:19.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:19.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:03:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:19.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:19.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:19.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:19.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:19.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:19.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:19.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:19.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:19.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:19.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:19.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:19.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:19.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:19.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=93, tag_tid=24, entry_tid=1' -a 'object_number=93, tag_tid=24, entry_tid=1' = 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=95, tag_tid=23, entry_tid=5903' '!=' 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:03:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:03:19.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:03:19.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:03:19.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:03:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:03:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:03:19.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:03:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:19.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:03:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:03:20.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:20.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:03:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:20.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:20.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:20.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:20.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:20.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:20.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:03:20.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:20.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:20.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:20.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:20.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:20.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:20.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:20.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:20.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=93, tag_tid=24, entry_tid=1' -a 'object_number=93, tag_tid=24, entry_tid=1' = 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=95, tag_tid=23, entry_tid=5903' '!=' 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:03:20.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:03:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:03:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:03:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:03:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:03:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:03:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:03:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:03:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:03:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:03:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:03:21.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:21.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:03:21.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:21.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:21.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:21.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:03:21.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:21.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:21.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:21.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:21.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902], [object_number=93, tag_tid=23, entry_tid=5901], [object_number=92, tag_tid=23, entry_tid=5900]]], state=connected] 2024-09-14T07:03:21.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=93, tag_tid=24, entry_tid=1' -a 'object_number=93, tag_tid=24, entry_tid=1' = 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=95, tag_tid=23, entry_tid=5903' '!=' 'object_number=95, tag_tid=23, entry_tid=5903' 2024-09-14T07:03:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:03:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:03:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:03:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:03:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:03:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:03:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:03:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:03:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:03:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:03:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:03:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:03:22.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:03:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:03:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:03:23.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:03:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:23.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:23.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:23.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:23.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:23.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:23.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:23.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:03:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:03:23.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:03:23.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:03:23.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:03:23.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:03:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 23 2024-09-14T07:03:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 23 2024-09-14T07:03:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:03:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=93, tag_tid=24, entry_tid=1], [object_number=92, tag_tid=24, entry_tid=0], [object_number=95, tag_tid=23, entry_tid=5903], [object_number=94, tag_tid=23, entry_tid=5902]]], state=connected] 2024-09-14T07:03:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=93, tag_tid=24, entry_tid=1' -a 'object_number=93, tag_tid=24, entry_tid=1' = 'object_number=93, tag_tid=24, entry_tid=1' 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8 2024-09-14T07:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:03:23.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:03:23.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:03:24.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2024-09-14T07:03:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:03:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:03:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:03:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 15 2024-09-14T07:03:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:02:35 2024 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:03:03 2024 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:03:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:03:24.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:03:24.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:24.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap7-clean ']' 2024-09-14T07:03:24.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap7-clean 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7-clean 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:24.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:03:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:03:24.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:26.086 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:03:26.088+0000 7f0de5b8b640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:03:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:03:26.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:26.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap8 2024-09-14T07:03:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8 2024-09-14T07:03:26.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:03:26.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:03:26.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:26.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:26.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:03:27.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:03:27.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:27.601 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:03:27.604+0000 7fea4c395640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:03:27.603 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:03:27.604+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:03:27.720 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:03:27.724+0000 7f99dda17640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:03:27.943 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:03:27.944+0000 7f99de218640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:03:27.944 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:03:27.944+0000 7f99e0a1d640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:03:27.945 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:03:27.948+0000 7f99dca15640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:03:28.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:03:28.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:29.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:03:29.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:29.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap8-clean 2024-09-14T07:03:29.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap8-clean 2024-09-14T07:03:29.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:03:29.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:29.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:29.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8-clean 2024-09-14T07:03:29.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T07:03:29.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:03:29.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T07:03:29.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:03:29.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:29.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:29.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:03:29.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:03:29.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:03:29.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T07:03:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 35093504 bytes 2024-09-14T07:03:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 128225280 bytes 2024-09-14T07:03:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 162987008 bytes 2024-09-14T07:03:39.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:03:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:03:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:03:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T07:03:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:03:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:03:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:03:39.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:03:39.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:03:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:03:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T07:03:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T07:03:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:03:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":1284927.73,"entries_behind_primary":6555,"entries_per_second":81.47,"non_primary_position":{"entry_tid":1,"object_number":93,"tag_tid":24},"primary_position":{"entry_tid":6554,"object_number":102,"tag_tid":26},"seconds_until_synced":80} 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:03:40 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:03:10' 2024-09-14T07:03:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T07:03:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T07:03:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T07:03:40.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":1284927.73,"entries_behind_primary":6555,"entries_per_second":81.47,"non_primary_position":{"entry_tid":1,"object_number":93,"tag_tid":24},"primary_position":{"entry_tid":6554,"object_number":102,"tag_tid":26},"seconds_until_synced":80} 2024-09-14T07:03:40.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:03:40.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:03:40.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T07:03:40.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:03:40.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T07:03:40.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T07:03:40.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T07:03:40.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T07:03:40.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:03:40' 2024-09-14T07:03:40.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 07:03:40' 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 07:03:40' 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:03:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T07:03:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK44074410mirror.2smithi16919.3.0-4775-g7f24e589trueOK44494452mirror.1smithi16919.3.0-4775-g7f24e589falseOK44794482mirror.3smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":1284927.73,"entries_behind_primary":6555,"entries_per_second":81.47,"non_primary_position":{"entry_tid":1,"object_number":93,"tag_tid":24},"primary_position":{"entry_tid":6554,"object_number":102,"tag_tid":26},"seconds_until_synced":80}42104261mirror.0smithi1692024-09-14 07:03:40cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 07:03:40' 2024-09-14T07:03:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T07:03:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T07:03:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:03:40' 2024-09-14T07:03:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T07:03:40.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T07:03:40.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:03:40.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T07:03:40.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:03:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:03:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap9 2024-09-14T07:03:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap9 2024-09-14T07:03:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:03:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:03:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:03:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9 2024-09-14T07:03:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:03:55.736 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8233412288159735) > weight (0.25) 2024-09-14T07:03:55.737 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T07:03:55.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T07:03:55.737 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T07:03:55.739 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:03:55.738+0000 7fb6a7816640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 60012) UID: 1000 2024-09-14T07:04:03.738 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T07:04:03.738 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:04:03.739 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T07:04:03.739 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T07:04:03.739 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T07:04:03.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T07:04:03.742 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:04:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:09.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:04:10.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T07:04:10.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:10.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:04:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:04:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:04:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:04:10.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:10.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:04:10.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:04:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:04:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:04:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:04:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:04:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:10.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:10.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:10.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:10.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:04:10.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:04:10.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:04:10.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:04:10.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:04:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:10.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:10.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:10.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:10.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:10.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:10.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:10.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:10.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:11.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:11.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:11.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:11.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:11.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:11.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:11.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:11.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:11.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:11.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=6953], [object_number=100, tag_tid=26, entry_tid=6952], [object_number=103, tag_tid=26, entry_tid=6951], [object_number=102, tag_tid=26, entry_tid=6950]]], state=connected] 2024-09-14T07:04:11.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=6953], [object_number=100, tag_tid=26, entry_tid=6952], [object_number=103, tag_tid=26, entry_tid=6951], [object_number=102, tag_tid=26, entry_tid=6950]]], state=connected] 2024-09-14T07:04:11.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=6953' '!=' '' 2024-09-14T07:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=6953' = '' 2024-09-14T07:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:11.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:11.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:11.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:11.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:11.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:11.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:11.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=6953 2024-09-14T07:04:11.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 6953 2024-09-14T07:04:11.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:11.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:11.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:11.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:11.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:11.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:11.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:11.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:11.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:11.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:11.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7063], [object_number=102, tag_tid=26, entry_tid=7062], [object_number=101, tag_tid=26, entry_tid=7061], [object_number=100, tag_tid=26, entry_tid=7060]]], state=connected] 2024-09-14T07:04:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:11.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:11.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:11.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:11.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7063], [object_number=102, tag_tid=26, entry_tid=7062], [object_number=101, tag_tid=26, entry_tid=7061], [object_number=100, tag_tid=26, entry_tid=7060]]], state=connected] 2024-09-14T07:04:11.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:11.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:11.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7063' '!=' 'object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7063' = 'object_number=101, tag_tid=26, entry_tid=6953' 2024-09-14T07:04:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7063 2024-09-14T07:04:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7063 2024-09-14T07:04:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:11.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:11.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:11.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:11.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:11.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:12.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:12.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:12.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:12.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:12.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:12.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:12.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=7157], [object_number=100, tag_tid=26, entry_tid=7156], [object_number=103, tag_tid=26, entry_tid=7155], [object_number=102, tag_tid=26, entry_tid=7154]]], state=connected] 2024-09-14T07:04:12.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=7157], [object_number=100, tag_tid=26, entry_tid=7156], [object_number=103, tag_tid=26, entry_tid=7155], [object_number=102, tag_tid=26, entry_tid=7154]]], state=connected] 2024-09-14T07:04:12.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=7157' '!=' 'object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=7157' = 'object_number=103, tag_tid=26, entry_tid=7063' 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:12.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:12.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:12.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:12.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:12.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:12.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:12.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7157 2024-09-14T07:04:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7157 2024-09-14T07:04:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:12.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:12.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:12.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:12.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:12.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:12.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:12.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:12.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7255], [object_number=102, tag_tid=26, entry_tid=7254], [object_number=101, tag_tid=26, entry_tid=7253], [object_number=100, tag_tid=26, entry_tid=7252]]], state=connected] 2024-09-14T07:04:12.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:12.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:12.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:12.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:12.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:12.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:12.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:12.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:12.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7255], [object_number=102, tag_tid=26, entry_tid=7254], [object_number=101, tag_tid=26, entry_tid=7253], [object_number=100, tag_tid=26, entry_tid=7252]]], state=connected] 2024-09-14T07:04:12.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7255' '!=' 'object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7255' = 'object_number=101, tag_tid=26, entry_tid=7157' 2024-09-14T07:04:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:12.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:12.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:12.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:12.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:12.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:12.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:12.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:12.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:12.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:12.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:12.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:12.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:12.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7255 2024-09-14T07:04:12.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7255 2024-09-14T07:04:12.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:12.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:12.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:12.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:12.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:12.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:13.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:13.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:13.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:13.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:13.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:13.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:13.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:13.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=7352], [object_number=103, tag_tid=26, entry_tid=7351], [object_number=102, tag_tid=26, entry_tid=7350], [object_number=101, tag_tid=26, entry_tid=7349]]], state=connected] 2024-09-14T07:04:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:13.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:13.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:13.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:13.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=7352], [object_number=103, tag_tid=26, entry_tid=7351], [object_number=102, tag_tid=26, entry_tid=7350], [object_number=101, tag_tid=26, entry_tid=7349]]], state=connected] 2024-09-14T07:04:13.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=7352' '!=' 'object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=7352' = 'object_number=103, tag_tid=26, entry_tid=7255' 2024-09-14T07:04:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:13.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:13.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:13.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7352 2024-09-14T07:04:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7352 2024-09-14T07:04:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:13.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:13.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:13.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:13.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:13.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:13.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:13.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:13.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:13.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:13.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:13.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:13.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:13.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:13.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:13.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:13.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:13.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:13.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:13.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=7453], [object_number=100, tag_tid=26, entry_tid=7452], [object_number=103, tag_tid=26, entry_tid=7451], [object_number=102, tag_tid=26, entry_tid=7450]]], state=connected] 2024-09-14T07:04:13.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=7453], [object_number=100, tag_tid=26, entry_tid=7452], [object_number=103, tag_tid=26, entry_tid=7451], [object_number=102, tag_tid=26, entry_tid=7450]]], state=connected] 2024-09-14T07:04:13.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:13.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:13.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:13.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=7453' '!=' 'object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:13.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=7453' = 'object_number=100, tag_tid=26, entry_tid=7352' 2024-09-14T07:04:13.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:13.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:13.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:13.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:13.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:13.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:13.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:13.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:13.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:13.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:13.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:13.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:13.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7453 2024-09-14T07:04:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7453 2024-09-14T07:04:13.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:13.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:13.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:13.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:13.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:13.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:13.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:13.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:13.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:14.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:14.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:14.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:14.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:14.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:14.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:14.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:14.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:14.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:14.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:14.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=7556], [object_number=103, tag_tid=26, entry_tid=7555], [object_number=102, tag_tid=26, entry_tid=7554], [object_number=101, tag_tid=26, entry_tid=7553]]], state=connected] 2024-09-14T07:04:14.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:14.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:14.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:14.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=7556], [object_number=103, tag_tid=26, entry_tid=7555], [object_number=102, tag_tid=26, entry_tid=7554], [object_number=101, tag_tid=26, entry_tid=7553]]], state=connected] 2024-09-14T07:04:14.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=7556' '!=' 'object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=7556' = 'object_number=101, tag_tid=26, entry_tid=7453' 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7556 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7556 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:14.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:14.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:14.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:14.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:14.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:14.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:14.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:14.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:14.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:14.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:14.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:14.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:14.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:14.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:14.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:14.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:14.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:14.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7663], [object_number=102, tag_tid=26, entry_tid=7662], [object_number=101, tag_tid=26, entry_tid=7661], [object_number=100, tag_tid=26, entry_tid=7660]]], state=connected] 2024-09-14T07:04:14.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:14.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:14.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7663], [object_number=102, tag_tid=26, entry_tid=7662], [object_number=101, tag_tid=26, entry_tid=7661], [object_number=100, tag_tid=26, entry_tid=7660]]], state=connected] 2024-09-14T07:04:14.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:14.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7663' '!=' 'object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7663' = 'object_number=100, tag_tid=26, entry_tid=7556' 2024-09-14T07:04:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:14.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:14.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:14.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:14.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:14.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:14.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:14.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:14.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:14.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:14.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7663 2024-09-14T07:04:14.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7663 2024-09-14T07:04:14.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:14.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:14.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:14.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:14.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:15.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=7762], [object_number=101, tag_tid=26, entry_tid=7761], [object_number=100, tag_tid=26, entry_tid=7760], [object_number=103, tag_tid=26, entry_tid=7759]]], state=connected] 2024-09-14T07:04:15.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:15.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:15.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:15.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:15.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:15.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:15.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=7762], [object_number=101, tag_tid=26, entry_tid=7761], [object_number=100, tag_tid=26, entry_tid=7760], [object_number=103, tag_tid=26, entry_tid=7759]]], state=connected] 2024-09-14T07:04:15.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=7762' '!=' 'object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=7762' = 'object_number=103, tag_tid=26, entry_tid=7663' 2024-09-14T07:04:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:15.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:15.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:15.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:15.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:15.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:15.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:15.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:15.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7762 2024-09-14T07:04:15.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7762 2024-09-14T07:04:15.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:15.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:15.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:15.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:15.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:15.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:15.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:15.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:15.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:15.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:15.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:15.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:15.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:15.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:15.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:15.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:15.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:15.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:15.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:15.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:15.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:15.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:15.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:15.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:15.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:15.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:15.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=7905], [object_number=100, tag_tid=26, entry_tid=7904], [object_number=103, tag_tid=26, entry_tid=7903], [object_number=102, tag_tid=26, entry_tid=7902]]], state=connected] 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=7905], [object_number=100, tag_tid=26, entry_tid=7904], [object_number=103, tag_tid=26, entry_tid=7903], [object_number=102, tag_tid=26, entry_tid=7902]]], state=connected] 2024-09-14T07:04:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=7905' '!=' 'object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=7905' = 'object_number=102, tag_tid=26, entry_tid=7762' 2024-09-14T07:04:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:15.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:15.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:15.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:15.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:15.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:15.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:15.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7905 2024-09-14T07:04:15.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7905 2024-09-14T07:04:15.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:15.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:15.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:16.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:16.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:16.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:16.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:16.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:16.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:16.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:16.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:16.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:16.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:16.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7999], [object_number=102, tag_tid=26, entry_tid=7998], [object_number=101, tag_tid=26, entry_tid=7997], [object_number=100, tag_tid=26, entry_tid=7996]]], state=connected] 2024-09-14T07:04:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:16.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:16.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:16.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:16.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=7999], [object_number=102, tag_tid=26, entry_tid=7998], [object_number=101, tag_tid=26, entry_tid=7997], [object_number=100, tag_tid=26, entry_tid=7996]]], state=connected] 2024-09-14T07:04:16.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7999' '!=' 'object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:16.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:16.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=7999' = 'object_number=101, tag_tid=26, entry_tid=7905' 2024-09-14T07:04:16.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:16.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:16.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:16.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:16.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:16.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:16.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:16.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:16.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:16.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7999 2024-09-14T07:04:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 7999 2024-09-14T07:04:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:16.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:16.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:16.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:16.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:16.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:16.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:16.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:16.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:16.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:16.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:16.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:16.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:16.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:16.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:16.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:16.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:16.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:16.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:16.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:16.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:16.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=8107], [object_number=102, tag_tid=26, entry_tid=8106], [object_number=101, tag_tid=26, entry_tid=8105], [object_number=100, tag_tid=26, entry_tid=8104]]], state=connected] 2024-09-14T07:04:16.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:16.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:16.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:16.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:16.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:16.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:16.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:16.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=8107], [object_number=102, tag_tid=26, entry_tid=8106], [object_number=101, tag_tid=26, entry_tid=8105], [object_number=100, tag_tid=26, entry_tid=8104]]], state=connected] 2024-09-14T07:04:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=8107' '!=' 'object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=8107' = 'object_number=103, tag_tid=26, entry_tid=7999' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8107 2024-09-14T07:04:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8107 2024-09-14T07:04:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:17.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:17.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:17.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:17.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:17.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:17.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:17.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=8214], [object_number=101, tag_tid=26, entry_tid=8213], [object_number=100, tag_tid=26, entry_tid=8212], [object_number=103, tag_tid=26, entry_tid=8211]]], state=connected] 2024-09-14T07:04:17.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:17.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:17.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:17.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:17.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:17.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:17.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:17.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:17.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=8214], [object_number=101, tag_tid=26, entry_tid=8213], [object_number=100, tag_tid=26, entry_tid=8212], [object_number=103, tag_tid=26, entry_tid=8211]]], state=connected] 2024-09-14T07:04:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=8214' '!=' 'object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=8214' = 'object_number=103, tag_tid=26, entry_tid=8107' 2024-09-14T07:04:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:17.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:17.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:17.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8214 2024-09-14T07:04:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8214 2024-09-14T07:04:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:17.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:17.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:17.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:17.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:17.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:17.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:17.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:17.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:17.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=8325], [object_number=100, tag_tid=26, entry_tid=8324], [object_number=103, tag_tid=26, entry_tid=8323], [object_number=102, tag_tid=26, entry_tid=8322]]], state=connected] 2024-09-14T07:04:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:17.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:17.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:17.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:17.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:17.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:17.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:17.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=8325], [object_number=100, tag_tid=26, entry_tid=8324], [object_number=103, tag_tid=26, entry_tid=8323], [object_number=102, tag_tid=26, entry_tid=8322]]], state=connected] 2024-09-14T07:04:17.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=8325' '!=' 'object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=8325' = 'object_number=102, tag_tid=26, entry_tid=8214' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:17.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:17.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:17.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:17.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:17.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:17.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:17.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8325 2024-09-14T07:04:17.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8325 2024-09-14T07:04:17.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:17.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:18.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:18.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:18.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:18.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:18.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:18.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:18.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:18.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:18.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:18.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:18.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:18.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:18.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:18.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:18.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:18.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:18.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:18.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:18.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:18.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:18.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:18.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:18.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:18.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=8419], [object_number=102, tag_tid=26, entry_tid=8418], [object_number=101, tag_tid=26, entry_tid=8417], [object_number=100, tag_tid=26, entry_tid=8416]]], state=connected] 2024-09-14T07:04:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:18.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:18.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:18.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=8419], [object_number=102, tag_tid=26, entry_tid=8418], [object_number=101, tag_tid=26, entry_tid=8417], [object_number=100, tag_tid=26, entry_tid=8416]]], state=connected] 2024-09-14T07:04:18.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=8419' '!=' 'object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=8419' = 'object_number=101, tag_tid=26, entry_tid=8325' 2024-09-14T07:04:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:18.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:18.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:18.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:18.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:18.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:18.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:18.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:18.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:18.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:18.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:18.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:18.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8419 2024-09-14T07:04:18.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8419 2024-09-14T07:04:18.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:18.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:18.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:18.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:18.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:18.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:18.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:18.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:18.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:18.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:18.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:18.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:18.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=8557], [object_number=100, tag_tid=26, entry_tid=8556], [object_number=103, tag_tid=26, entry_tid=8555], [object_number=102, tag_tid=26, entry_tid=8554]]], state=connected] 2024-09-14T07:04:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=8557], [object_number=100, tag_tid=26, entry_tid=8556], [object_number=103, tag_tid=26, entry_tid=8555], [object_number=102, tag_tid=26, entry_tid=8554]]], state=connected] 2024-09-14T07:04:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=8557' '!=' 'object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=8557' = 'object_number=103, tag_tid=26, entry_tid=8419' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:19.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:19.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8557 2024-09-14T07:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8557 2024-09-14T07:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:19.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:19.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:19.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:19.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:19.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:19.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:19.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:19.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:19.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:19.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=8653], [object_number=100, tag_tid=26, entry_tid=8652], [object_number=103, tag_tid=26, entry_tid=8651], [object_number=102, tag_tid=26, entry_tid=8650]]], state=connected] 2024-09-14T07:04:19.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=8653], [object_number=100, tag_tid=26, entry_tid=8652], [object_number=103, tag_tid=26, entry_tid=8651], [object_number=102, tag_tid=26, entry_tid=8650]]], state=connected] 2024-09-14T07:04:19.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:19.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=8653' '!=' 'object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=8653' = 'object_number=101, tag_tid=26, entry_tid=8557' 2024-09-14T07:04:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:19.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:19.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:19.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:19.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8653 2024-09-14T07:04:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8653 2024-09-14T07:04:19.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:19.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:19.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:19.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:19.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:19.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:19.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:19.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:19.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:19.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:19.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:19.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:19.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:19.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:19.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:19.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:19.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:19.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:19.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:19.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:19.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=8743], [object_number=102, tag_tid=26, entry_tid=8742], [object_number=101, tag_tid=26, entry_tid=8741], [object_number=100, tag_tid=26, entry_tid=8740]]], state=connected] 2024-09-14T07:04:19.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:19.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:20.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:20.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:20.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:20.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:20.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=8743], [object_number=102, tag_tid=26, entry_tid=8742], [object_number=101, tag_tid=26, entry_tid=8741], [object_number=100, tag_tid=26, entry_tid=8740]]], state=connected] 2024-09-14T07:04:20.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=8743' '!=' 'object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=8743' = 'object_number=101, tag_tid=26, entry_tid=8653' 2024-09-14T07:04:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:20.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:20.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:20.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:20.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:20.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:20.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:20.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8743 2024-09-14T07:04:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8743 2024-09-14T07:04:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:20.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:20.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:20.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:20.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:20.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=8834], [object_number=101, tag_tid=26, entry_tid=8833], [object_number=100, tag_tid=26, entry_tid=8832], [object_number=103, tag_tid=26, entry_tid=8831]]], state=connected] 2024-09-14T07:04:20.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:20.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:20.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:20.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:20.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=8834], [object_number=101, tag_tid=26, entry_tid=8833], [object_number=100, tag_tid=26, entry_tid=8832], [object_number=103, tag_tid=26, entry_tid=8831]]], state=connected] 2024-09-14T07:04:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=8834' '!=' 'object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=8834' = 'object_number=103, tag_tid=26, entry_tid=8743' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:20.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:20.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8834 2024-09-14T07:04:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8834 2024-09-14T07:04:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:20.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:20.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:20.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:20.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:20.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:20.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:20.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:20.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:20.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:20.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:20.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=8932], [object_number=103, tag_tid=26, entry_tid=8931], [object_number=102, tag_tid=26, entry_tid=8930], [object_number=101, tag_tid=26, entry_tid=8929]]], state=connected] 2024-09-14T07:04:20.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=8932], [object_number=103, tag_tid=26, entry_tid=8931], [object_number=102, tag_tid=26, entry_tid=8930], [object_number=101, tag_tid=26, entry_tid=8929]]], state=connected] 2024-09-14T07:04:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=8932' '!=' 'object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=8932' = 'object_number=102, tag_tid=26, entry_tid=8834' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8932 2024-09-14T07:04:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 8932 2024-09-14T07:04:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:21.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:21.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:21.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:21.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:21.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:21.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:21.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:21.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:21.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:21.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:21.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:21.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:21.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:21.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:21.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=9037], [object_number=100, tag_tid=26, entry_tid=9036], [object_number=103, tag_tid=26, entry_tid=9035], [object_number=102, tag_tid=26, entry_tid=9034]]], state=connected] 2024-09-14T07:04:21.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=101, tag_tid=26, entry_tid=9037], [object_number=100, tag_tid=26, entry_tid=9036], [object_number=103, tag_tid=26, entry_tid=9035], [object_number=102, tag_tid=26, entry_tid=9034]]], state=connected] 2024-09-14T07:04:21.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:21.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:21.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=9037' '!=' 'object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=101, tag_tid=26, entry_tid=9037' = 'object_number=100, tag_tid=26, entry_tid=8932' 2024-09-14T07:04:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:21.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:21.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:21.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:21.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9037 2024-09-14T07:04:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9037 2024-09-14T07:04:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:21.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:21.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:21.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:21.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:21.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:21.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:21.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:21.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:21.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:21.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:21.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:21.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:21.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:21.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=9131], [object_number=102, tag_tid=26, entry_tid=9130], [object_number=101, tag_tid=26, entry_tid=9129], [object_number=100, tag_tid=26, entry_tid=9128]]], state=connected] 2024-09-14T07:04:21.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=9131], [object_number=102, tag_tid=26, entry_tid=9130], [object_number=101, tag_tid=26, entry_tid=9129], [object_number=100, tag_tid=26, entry_tid=9128]]], state=connected] 2024-09-14T07:04:22.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=9131' '!=' 'object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=9131' = 'object_number=101, tag_tid=26, entry_tid=9037' 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:22.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:22.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:22.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:22.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:22.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:22.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:22.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9131 2024-09-14T07:04:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9131 2024-09-14T07:04:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:22.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:22.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:22.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:22.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:22.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:22.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:22.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:22.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:22.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:22.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:22.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:22.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:22.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:22.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:22.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:22.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:22.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:22.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:22.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=9226], [object_number=101, tag_tid=26, entry_tid=9225], [object_number=100, tag_tid=26, entry_tid=9224], [object_number=103, tag_tid=26, entry_tid=9223]]], state=connected] 2024-09-14T07:04:22.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:22.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:22.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:22.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:22.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:22.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=9226], [object_number=101, tag_tid=26, entry_tid=9225], [object_number=100, tag_tid=26, entry_tid=9224], [object_number=103, tag_tid=26, entry_tid=9223]]], state=connected] 2024-09-14T07:04:22.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=9226' '!=' 'object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=9226' = 'object_number=103, tag_tid=26, entry_tid=9131' 2024-09-14T07:04:22.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:22.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:22.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:22.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:22.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9226 2024-09-14T07:04:22.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9226 2024-09-14T07:04:22.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:22.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:22.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:22.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:23.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:23.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:23.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:23.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:23.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:23.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=9355], [object_number=102, tag_tid=26, entry_tid=9354], [object_number=101, tag_tid=26, entry_tid=9353], [object_number=100, tag_tid=26, entry_tid=9352]]], state=connected] 2024-09-14T07:04:23.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:23.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:23.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:23.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:23.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=9355], [object_number=102, tag_tid=26, entry_tid=9354], [object_number=101, tag_tid=26, entry_tid=9353], [object_number=100, tag_tid=26, entry_tid=9352]]], state=connected] 2024-09-14T07:04:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=9355' '!=' 'object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=9355' = 'object_number=102, tag_tid=26, entry_tid=9226' 2024-09-14T07:04:23.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:23.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9355 2024-09-14T07:04:23.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9355 2024-09-14T07:04:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:23.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:23.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:23.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:23.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:23.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:23.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:23.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:23.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:23.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:23.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=9484], [object_number=103, tag_tid=26, entry_tid=9483], [object_number=102, tag_tid=26, entry_tid=9482], [object_number=101, tag_tid=26, entry_tid=9481]]], state=connected] 2024-09-14T07:04:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:23.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:23.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=9484], [object_number=103, tag_tid=26, entry_tid=9483], [object_number=102, tag_tid=26, entry_tid=9482], [object_number=101, tag_tid=26, entry_tid=9481]]], state=connected] 2024-09-14T07:04:23.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:23.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=9484' '!=' 'object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=9484' = 'object_number=103, tag_tid=26, entry_tid=9355' 2024-09-14T07:04:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:23.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:23.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:23.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:23.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9484 2024-09-14T07:04:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9484 2024-09-14T07:04:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:24.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:24.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:24.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:24.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:24.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:24.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:24.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:24.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=9575], [object_number=102, tag_tid=26, entry_tid=9574], [object_number=101, tag_tid=26, entry_tid=9573], [object_number=100, tag_tid=26, entry_tid=9572]]], state=connected] 2024-09-14T07:04:24.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:24.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:24.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:24.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=103, tag_tid=26, entry_tid=9575], [object_number=102, tag_tid=26, entry_tid=9574], [object_number=101, tag_tid=26, entry_tid=9573], [object_number=100, tag_tid=26, entry_tid=9572]]], state=connected] 2024-09-14T07:04:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=9575' '!=' 'object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=103, tag_tid=26, entry_tid=9575' = 'object_number=100, tag_tid=26, entry_tid=9484' 2024-09-14T07:04:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:24.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:24.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:24.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:24.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:24.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9575 2024-09-14T07:04:24.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9575 2024-09-14T07:04:24.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:24.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:24.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:24.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:24.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:24.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:24.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:24.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:24.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:24.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:24.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:24.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:24.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.837 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8752286846530697) > weight (0.25) 2024-09-14T07:04:24.837 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T07:04:24.838 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T07:04:24.838 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-14T07:04:24.839 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:04:24.837+0000 7fd9785be640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 68618) UID: 1000 2024-09-14T07:04:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:24.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=9678], [object_number=101, tag_tid=26, entry_tid=9677], [object_number=100, tag_tid=26, entry_tid=9676], [object_number=103, tag_tid=26, entry_tid=9675]]], state=connected] 2024-09-14T07:04:24.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:24.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:24.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=9678], [object_number=101, tag_tid=26, entry_tid=9677], [object_number=100, tag_tid=26, entry_tid=9676], [object_number=103, tag_tid=26, entry_tid=9675]]], state=connected] 2024-09-14T07:04:24.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:24.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:24.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:24.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=9678' '!=' 'object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:24.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=9678' = 'object_number=103, tag_tid=26, entry_tid=9575' 2024-09-14T07:04:24.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:24.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:24.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:24.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:24.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:24.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:24.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:24.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9678 2024-09-14T07:04:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9678 2024-09-14T07:04:24.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:24.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:24.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:25.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:25.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:25.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:25.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:25.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:25.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=9790], [object_number=101, tag_tid=26, entry_tid=9789], [object_number=100, tag_tid=26, entry_tid=9788], [object_number=103, tag_tid=26, entry_tid=9787]]], state=connected] 2024-09-14T07:04:25.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:25.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:25.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=102, tag_tid=26, entry_tid=9790], [object_number=101, tag_tid=26, entry_tid=9789], [object_number=100, tag_tid=26, entry_tid=9788], [object_number=103, tag_tid=26, entry_tid=9787]]], state=connected] 2024-09-14T07:04:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=9790' '!=' 'object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=102, tag_tid=26, entry_tid=9790' = 'object_number=102, tag_tid=26, entry_tid=9678' 2024-09-14T07:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:25.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:25.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:25.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:25.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:25.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9790 2024-09-14T07:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9790 2024-09-14T07:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:25.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:25.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:25.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:25.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:25.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:25.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:25.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:25.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:25.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:25.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:25.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:25.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:25.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:25.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:25.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:25.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:25.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:25.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=9892], [object_number=103, tag_tid=26, entry_tid=9891], [object_number=102, tag_tid=26, entry_tid=9890], [object_number=101, tag_tid=26, entry_tid=9889]]], state=connected] 2024-09-14T07:04:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 25 2024-09-14T07:04:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=100, tag_tid=26, entry_tid=9892], [object_number=103, tag_tid=26, entry_tid=9891], [object_number=102, tag_tid=26, entry_tid=9890], [object_number=101, tag_tid=26, entry_tid=9889]]], state=connected] 2024-09-14T07:04:25.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=9892' '!=' 'object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=100, tag_tid=26, entry_tid=9892' = 'object_number=102, tag_tid=26, entry_tid=9790' 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:25.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:25.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:25.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9892 2024-09-14T07:04:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9892 2024-09-14T07:04:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:26.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:26.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:26.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:26.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:26.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:26.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:26.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:26.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:26.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:26.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=9959], [object_number=106, tag_tid=26, entry_tid=9958], [object_number=105, tag_tid=26, entry_tid=9957], [object_number=104, tag_tid=26, entry_tid=9956]]], state=connected] 2024-09-14T07:04:26.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:26.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:26.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:26.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=9959], [object_number=106, tag_tid=26, entry_tid=9958], [object_number=105, tag_tid=26, entry_tid=9957], [object_number=104, tag_tid=26, entry_tid=9956]]], state=connected] 2024-09-14T07:04:26.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=9959' '!=' 'object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=9959' = 'object_number=100, tag_tid=26, entry_tid=9892' 2024-09-14T07:04:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:26.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:26.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9959 2024-09-14T07:04:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 9959 2024-09-14T07:04:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:26.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:26.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:26.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:26.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:26.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:26.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:26.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:26.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:26.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:26.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:26.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:26.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:26.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:26.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:26.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:26.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=10046], [object_number=105, tag_tid=26, entry_tid=10045], [object_number=104, tag_tid=26, entry_tid=10044], [object_number=107, tag_tid=26, entry_tid=10043]]], state=connected] 2024-09-14T07:04:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:26.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:26.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:26.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:26.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:26.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=10046], [object_number=105, tag_tid=26, entry_tid=10045], [object_number=104, tag_tid=26, entry_tid=10044], [object_number=107, tag_tid=26, entry_tid=10043]]], state=connected] 2024-09-14T07:04:26.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=10046' '!=' 'object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=10046' = 'object_number=107, tag_tid=26, entry_tid=9959' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:26.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:26.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:26.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:26.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:26.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:26.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:26.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:26.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:26.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:26.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:26.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10046 2024-09-14T07:04:26.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10046 2024-09-14T07:04:26.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:27.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:27.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:27.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:27.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:27.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:27.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:27.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:27.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:27.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:27.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:27.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:27.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:27.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:27.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10137], [object_number=104, tag_tid=26, entry_tid=10136], [object_number=107, tag_tid=26, entry_tid=10135], [object_number=106, tag_tid=26, entry_tid=10134]]], state=connected] 2024-09-14T07:04:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:27.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:27.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:27.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:27.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:27.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:27.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10137], [object_number=104, tag_tid=26, entry_tid=10136], [object_number=107, tag_tid=26, entry_tid=10135], [object_number=106, tag_tid=26, entry_tid=10134]]], state=connected] 2024-09-14T07:04:27.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10137' '!=' 'object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:27.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:27.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10137' = 'object_number=106, tag_tid=26, entry_tid=10046' 2024-09-14T07:04:27.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:27.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:27.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:27.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10137 2024-09-14T07:04:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10137 2024-09-14T07:04:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:27.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:27.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:27.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:27.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:27.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:27.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:27.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:27.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:27.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:27.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:27.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:27.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:27.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:27.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:27.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:27.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:27.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:27.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:27.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=10228], [object_number=107, tag_tid=26, entry_tid=10227], [object_number=106, tag_tid=26, entry_tid=10226], [object_number=105, tag_tid=26, entry_tid=10225]]], state=connected] 2024-09-14T07:04:27.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:27.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:27.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=10228], [object_number=107, tag_tid=26, entry_tid=10227], [object_number=106, tag_tid=26, entry_tid=10226], [object_number=105, tag_tid=26, entry_tid=10225]]], state=connected] 2024-09-14T07:04:27.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:27.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:27.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:27.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=10228' '!=' 'object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=10228' = 'object_number=105, tag_tid=26, entry_tid=10137' 2024-09-14T07:04:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:27.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:27.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:27.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:27.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:27.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:27.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:27.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:27.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:27.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:27.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:27.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10228 2024-09-14T07:04:27.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10228 2024-09-14T07:04:27.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:27.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:27.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:28.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:28.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:28.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:28.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:28.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:28.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:28.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:28.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:28.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:28.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:28.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:28.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=10319], [object_number=106, tag_tid=26, entry_tid=10318], [object_number=105, tag_tid=26, entry_tid=10317], [object_number=104, tag_tid=26, entry_tid=10316]]], state=connected] 2024-09-14T07:04:28.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:28.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:28.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:28.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=10319], [object_number=106, tag_tid=26, entry_tid=10318], [object_number=105, tag_tid=26, entry_tid=10317], [object_number=104, tag_tid=26, entry_tid=10316]]], state=connected] 2024-09-14T07:04:28.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=10319' '!=' 'object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=10319' = 'object_number=104, tag_tid=26, entry_tid=10228' 2024-09-14T07:04:28.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:28.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:28.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:28.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:28.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:28.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:28.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:28.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10319 2024-09-14T07:04:28.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10319 2024-09-14T07:04:28.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:28.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:28.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:28.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:28.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:28.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=10415], [object_number=106, tag_tid=26, entry_tid=10414], [object_number=105, tag_tid=26, entry_tid=10413], [object_number=104, tag_tid=26, entry_tid=10412]]], state=connected] 2024-09-14T07:04:28.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:28.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:28.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:28.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:28.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:28.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:28.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=10415], [object_number=106, tag_tid=26, entry_tid=10414], [object_number=105, tag_tid=26, entry_tid=10413], [object_number=104, tag_tid=26, entry_tid=10412]]], state=connected] 2024-09-14T07:04:28.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:28.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:28.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:28.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=10415' '!=' 'object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:28.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=10415' = 'object_number=107, tag_tid=26, entry_tid=10319' 2024-09-14T07:04:28.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10415 2024-09-14T07:04:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10415 2024-09-14T07:04:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:29.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:29.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:29.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:29.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:29.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:29.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:29.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:29.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:29.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:29.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=10516], [object_number=107, tag_tid=26, entry_tid=10515], [object_number=106, tag_tid=26, entry_tid=10514], [object_number=105, tag_tid=26, entry_tid=10513]]], state=connected] 2024-09-14T07:04:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:29.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:29.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:29.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:29.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:29.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:29.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:29.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=10516], [object_number=107, tag_tid=26, entry_tid=10515], [object_number=106, tag_tid=26, entry_tid=10514], [object_number=105, tag_tid=26, entry_tid=10513]]], state=connected] 2024-09-14T07:04:29.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=10516' '!=' 'object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=10516' = 'object_number=107, tag_tid=26, entry_tid=10415' 2024-09-14T07:04:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:29.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:29.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:29.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:29.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:29.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:29.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:29.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:29.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10516 2024-09-14T07:04:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10516 2024-09-14T07:04:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:29.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:29.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:29.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:29.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:29.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:29.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:29.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:29.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:29.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:29.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:29.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:29.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:29.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:29.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:29.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:29.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:29.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:29.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:29.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:29.839 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T07:04:29.839 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:04:29.839 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T07:04:29.840 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T07:04:29.840 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T07:04:29.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T07:04:29.842 INFO:tasks.rbd_mirror_thrash:waiting for 80 secs before thrashing 2024-09-14T07:04:30.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:30.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:30.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10609], [object_number=104, tag_tid=26, entry_tid=10608], [object_number=107, tag_tid=26, entry_tid=10607], [object_number=106, tag_tid=26, entry_tid=10606]]], state=connected] 2024-09-14T07:04:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:30.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:30.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:30.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:30.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:30.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:30.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:30.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10609], [object_number=104, tag_tid=26, entry_tid=10608], [object_number=107, tag_tid=26, entry_tid=10607], [object_number=106, tag_tid=26, entry_tid=10606]]], state=connected] 2024-09-14T07:04:30.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10609' '!=' 'object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10609' = 'object_number=104, tag_tid=26, entry_tid=10516' 2024-09-14T07:04:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:30.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:30.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:30.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:30.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:30.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:30.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:30.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10609 2024-09-14T07:04:30.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10609 2024-09-14T07:04:30.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:30.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:30.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:30.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:30.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:30.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:30.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:30.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:30.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:30.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=10738], [object_number=105, tag_tid=26, entry_tid=10737], [object_number=104, tag_tid=26, entry_tid=10736], [object_number=107, tag_tid=26, entry_tid=10735]]], state=connected] 2024-09-14T07:04:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:30.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:30.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:30.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:30.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:30.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:30.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:30.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=10738], [object_number=105, tag_tid=26, entry_tid=10737], [object_number=104, tag_tid=26, entry_tid=10736], [object_number=107, tag_tid=26, entry_tid=10735]]], state=connected] 2024-09-14T07:04:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=10738' '!=' 'object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=10738' = 'object_number=105, tag_tid=26, entry_tid=10609' 2024-09-14T07:04:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:30.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:30.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:30.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:30.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:30.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:30.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:30.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:30.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:30.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10738 2024-09-14T07:04:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10738 2024-09-14T07:04:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:30.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:30.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:30.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:30.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:30.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:30.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:30.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:30.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:30.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:30.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10821], [object_number=104, tag_tid=26, entry_tid=10820], [object_number=107, tag_tid=26, entry_tid=10819], [object_number=106, tag_tid=26, entry_tid=10818]]], state=connected] 2024-09-14T07:04:30.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:30.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:30.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:30.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:30.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:31.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:31.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:31.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:31.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:31.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:31.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10821], [object_number=104, tag_tid=26, entry_tid=10820], [object_number=107, tag_tid=26, entry_tid=10819], [object_number=106, tag_tid=26, entry_tid=10818]]], state=connected] 2024-09-14T07:04:31.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10821' '!=' 'object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10821' = 'object_number=106, tag_tid=26, entry_tid=10738' 2024-09-14T07:04:31.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:31.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:31.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:31.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:31.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10821 2024-09-14T07:04:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10821 2024-09-14T07:04:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:31.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:31.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:31.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:31.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:31.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:31.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:31.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:31.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:31.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:31.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:31.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:31.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:31.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10925], [object_number=104, tag_tid=26, entry_tid=10924], [object_number=107, tag_tid=26, entry_tid=10923], [object_number=106, tag_tid=26, entry_tid=10922]]], state=connected] 2024-09-14T07:04:31.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:31.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:31.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:31.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:31.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=10925], [object_number=104, tag_tid=26, entry_tid=10924], [object_number=107, tag_tid=26, entry_tid=10923], [object_number=106, tag_tid=26, entry_tid=10922]]], state=connected] 2024-09-14T07:04:31.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:31.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:31.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10925' '!=' 'object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:31.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=10925' = 'object_number=105, tag_tid=26, entry_tid=10821' 2024-09-14T07:04:31.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:31.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:31.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:31.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10925 2024-09-14T07:04:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 10925 2024-09-14T07:04:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:31.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:31.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:31.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:31.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:31.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:31.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:31.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:31.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:31.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:31.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:31.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:31.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:31.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:31.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:31.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:31.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:31.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:31.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:31.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:31.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:31.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:31.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:31.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:31.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:31.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:31.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:31.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:31.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=11028], [object_number=107, tag_tid=26, entry_tid=11027], [object_number=106, tag_tid=26, entry_tid=11026], [object_number=105, tag_tid=26, entry_tid=11025]]], state=connected] 2024-09-14T07:04:31.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:31.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:31.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:31.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:31.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:31.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=11028], [object_number=107, tag_tid=26, entry_tid=11027], [object_number=106, tag_tid=26, entry_tid=11026], [object_number=105, tag_tid=26, entry_tid=11025]]], state=connected] 2024-09-14T07:04:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=11028' '!=' 'object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=11028' = 'object_number=105, tag_tid=26, entry_tid=10925' 2024-09-14T07:04:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:32.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:32.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11028 2024-09-14T07:04:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11028 2024-09-14T07:04:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:32.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:32.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:32.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:32.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:32.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:32.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:32.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:32.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:32.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:32.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:32.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:32.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:32.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:32.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11119], [object_number=106, tag_tid=26, entry_tid=11118], [object_number=105, tag_tid=26, entry_tid=11117], [object_number=104, tag_tid=26, entry_tid=11116]]], state=connected] 2024-09-14T07:04:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:32.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:32.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:32.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:32.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:32.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11119], [object_number=106, tag_tid=26, entry_tid=11118], [object_number=105, tag_tid=26, entry_tid=11117], [object_number=104, tag_tid=26, entry_tid=11116]]], state=connected] 2024-09-14T07:04:32.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:32.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:32.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11119' '!=' 'object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11119' = 'object_number=104, tag_tid=26, entry_tid=11028' 2024-09-14T07:04:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:32.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:32.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:32.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:32.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:32.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:32.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:32.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:32.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:32.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11119 2024-09-14T07:04:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11119 2024-09-14T07:04:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:32.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:32.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:32.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:32.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:32.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:32.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:32.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:32.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:32.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:32.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:32.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:32.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:32.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:32.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:32.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:32.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:32.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:32.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:32.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:32.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:32.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:32.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:32.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:32.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:32.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:32.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:32.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:32.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=11213], [object_number=104, tag_tid=26, entry_tid=11212], [object_number=107, tag_tid=26, entry_tid=11211], [object_number=106, tag_tid=26, entry_tid=11210]]], state=connected] 2024-09-14T07:04:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:32.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:32.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:32.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:33.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:33.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:33.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:33.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=11213], [object_number=104, tag_tid=26, entry_tid=11212], [object_number=107, tag_tid=26, entry_tid=11211], [object_number=106, tag_tid=26, entry_tid=11210]]], state=connected] 2024-09-14T07:04:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=11213' '!=' 'object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=11213' = 'object_number=107, tag_tid=26, entry_tid=11119' 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:33.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:33.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:33.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:33.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:33.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:33.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:33.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:33.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11213 2024-09-14T07:04:33.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11213 2024-09-14T07:04:33.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:33.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:33.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:33.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:33.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:33.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:33.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:33.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:33.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:33.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:33.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=11302], [object_number=105, tag_tid=26, entry_tid=11301], [object_number=104, tag_tid=26, entry_tid=11300], [object_number=107, tag_tid=26, entry_tid=11299]]], state=connected] 2024-09-14T07:04:33.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=11302], [object_number=105, tag_tid=26, entry_tid=11301], [object_number=104, tag_tid=26, entry_tid=11300], [object_number=107, tag_tid=26, entry_tid=11299]]], state=connected] 2024-09-14T07:04:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=11302' '!=' 'object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=11302' = 'object_number=105, tag_tid=26, entry_tid=11213' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:33.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:33.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:33.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:33.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:33.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:33.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:33.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:33.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:33.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11302 2024-09-14T07:04:33.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11302 2024-09-14T07:04:33.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:33.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:33.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:33.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:33.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:33.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:33.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:33.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:33.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:33.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:33.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:33.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=11398], [object_number=105, tag_tid=26, entry_tid=11397], [object_number=104, tag_tid=26, entry_tid=11396], [object_number=107, tag_tid=26, entry_tid=11395]]], state=connected] 2024-09-14T07:04:33.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:33.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:33.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:33.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:33.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:34.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:34.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:34.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:34.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=106, tag_tid=26, entry_tid=11398], [object_number=105, tag_tid=26, entry_tid=11397], [object_number=104, tag_tid=26, entry_tid=11396], [object_number=107, tag_tid=26, entry_tid=11395]]], state=connected] 2024-09-14T07:04:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=11398' '!=' 'object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=106, tag_tid=26, entry_tid=11398' = 'object_number=106, tag_tid=26, entry_tid=11302' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:34.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:34.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:34.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:34.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11398 2024-09-14T07:04:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11398 2024-09-14T07:04:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:34.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:34.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:34.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:34.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:34.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:34.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:34.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:34.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:34.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:34.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:34.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:34.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:34.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:34.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:34.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:34.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:34.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11483], [object_number=106, tag_tid=26, entry_tid=11482], [object_number=105, tag_tid=26, entry_tid=11481], [object_number=104, tag_tid=26, entry_tid=11480]]], state=connected] 2024-09-14T07:04:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:34.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:34.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:34.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:34.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:34.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:34.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:34.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:34.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:34.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11483], [object_number=106, tag_tid=26, entry_tid=11482], [object_number=105, tag_tid=26, entry_tid=11481], [object_number=104, tag_tid=26, entry_tid=11480]]], state=connected] 2024-09-14T07:04:34.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11483' '!=' 'object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11483' = 'object_number=106, tag_tid=26, entry_tid=11398' 2024-09-14T07:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:34.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:34.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:34.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:34.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:34.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:34.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11483 2024-09-14T07:04:34.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11483 2024-09-14T07:04:34.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:34.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:34.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:34.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:34.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:34.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:34.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:34.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:34.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:34.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:34.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:34.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:34.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:34.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:34.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11571], [object_number=106, tag_tid=26, entry_tid=11570], [object_number=105, tag_tid=26, entry_tid=11569], [object_number=104, tag_tid=26, entry_tid=11568]]], state=connected] 2024-09-14T07:04:34.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:34.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:35.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:35.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:35.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:35.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11571], [object_number=106, tag_tid=26, entry_tid=11570], [object_number=105, tag_tid=26, entry_tid=11569], [object_number=104, tag_tid=26, entry_tid=11568]]], state=connected] 2024-09-14T07:04:35.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11571' '!=' 'object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11571' = 'object_number=107, tag_tid=26, entry_tid=11483' 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:35.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:35.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:35.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:35.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11571 2024-09-14T07:04:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11571 2024-09-14T07:04:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:35.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:35.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:35.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:35.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:35.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:35.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:35.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:35.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:35.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:35.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:35.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=11656], [object_number=107, tag_tid=26, entry_tid=11655], [object_number=106, tag_tid=26, entry_tid=11654], [object_number=105, tag_tid=26, entry_tid=11653]]], state=connected] 2024-09-14T07:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=11656], [object_number=107, tag_tid=26, entry_tid=11655], [object_number=106, tag_tid=26, entry_tid=11654], [object_number=105, tag_tid=26, entry_tid=11653]]], state=connected] 2024-09-14T07:04:35.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:35.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:35.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=11656' '!=' 'object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:35.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=11656' = 'object_number=107, tag_tid=26, entry_tid=11571' 2024-09-14T07:04:35.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:35.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:35.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:35.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:35.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:35.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:35.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11656 2024-09-14T07:04:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11656 2024-09-14T07:04:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:35.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:35.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:35.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:35.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:35.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:35.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:35.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:35.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:35.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:35.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:35.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:35.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:35.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:35.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:35.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:35.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11739], [object_number=106, tag_tid=26, entry_tid=11738], [object_number=105, tag_tid=26, entry_tid=11737], [object_number=104, tag_tid=26, entry_tid=11736]]], state=connected] 2024-09-14T07:04:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:35.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:35.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:36.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:36.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:36.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:36.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11739], [object_number=106, tag_tid=26, entry_tid=11738], [object_number=105, tag_tid=26, entry_tid=11737], [object_number=104, tag_tid=26, entry_tid=11736]]], state=connected] 2024-09-14T07:04:36.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11739' '!=' 'object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11739' = 'object_number=104, tag_tid=26, entry_tid=11656' 2024-09-14T07:04:36.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:36.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:36.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11739 2024-09-14T07:04:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11739 2024-09-14T07:04:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:36.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:36.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:36.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:36.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:36.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:36.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:36.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:36.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:36.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:36.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:36.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:36.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:36.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:36.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:36.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:36.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:36.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:36.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:36.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:36.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=11821], [object_number=104, tag_tid=26, entry_tid=11820], [object_number=107, tag_tid=26, entry_tid=11819], [object_number=106, tag_tid=26, entry_tid=11818]]], state=connected] 2024-09-14T07:04:36.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=11821], [object_number=104, tag_tid=26, entry_tid=11820], [object_number=107, tag_tid=26, entry_tid=11819], [object_number=106, tag_tid=26, entry_tid=11818]]], state=connected] 2024-09-14T07:04:36.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=11821' '!=' 'object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=11821' = 'object_number=107, tag_tid=26, entry_tid=11739' 2024-09-14T07:04:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:36.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11821 2024-09-14T07:04:36.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11821 2024-09-14T07:04:36.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:36.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:36.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:37.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:37.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:37.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:37.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:37.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:37.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11939], [object_number=106, tag_tid=26, entry_tid=11938], [object_number=105, tag_tid=26, entry_tid=11937], [object_number=104, tag_tid=26, entry_tid=11936]]], state=connected] 2024-09-14T07:04:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:37.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:37.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:37.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:37.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:37.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:37.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:37.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=11939], [object_number=106, tag_tid=26, entry_tid=11938], [object_number=105, tag_tid=26, entry_tid=11937], [object_number=104, tag_tid=26, entry_tid=11936]]], state=connected] 2024-09-14T07:04:37.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11939' '!=' 'object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=11939' = 'object_number=105, tag_tid=26, entry_tid=11821' 2024-09-14T07:04:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11939 2024-09-14T07:04:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 11939 2024-09-14T07:04:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:37.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:37.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:37.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:37.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:37.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:37.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:37.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:37.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:37.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:37.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:37.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=12023], [object_number=106, tag_tid=26, entry_tid=12022], [object_number=105, tag_tid=26, entry_tid=12021], [object_number=104, tag_tid=26, entry_tid=12020]]], state=connected] 2024-09-14T07:04:37.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:37.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=12023], [object_number=106, tag_tid=26, entry_tid=12022], [object_number=105, tag_tid=26, entry_tid=12021], [object_number=104, tag_tid=26, entry_tid=12020]]], state=connected] 2024-09-14T07:04:37.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:37.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:37.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=12023' '!=' 'object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=12023' = 'object_number=107, tag_tid=26, entry_tid=11939' 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:37.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:37.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:37.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12023 2024-09-14T07:04:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12023 2024-09-14T07:04:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:37.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:37.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:37.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:38.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:38.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:38.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:38.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:38.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:38.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=12111], [object_number=106, tag_tid=26, entry_tid=12110], [object_number=105, tag_tid=26, entry_tid=12109], [object_number=104, tag_tid=26, entry_tid=12108]]], state=connected] 2024-09-14T07:04:38.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:38.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:38.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:38.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:38.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:38.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=12111], [object_number=106, tag_tid=26, entry_tid=12110], [object_number=105, tag_tid=26, entry_tid=12109], [object_number=104, tag_tid=26, entry_tid=12108]]], state=connected] 2024-09-14T07:04:38.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=12111' '!=' 'object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:38.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=12111' = 'object_number=107, tag_tid=26, entry_tid=12023' 2024-09-14T07:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:38.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:38.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:38.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:38.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:38.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:38.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:38.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:38.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12111 2024-09-14T07:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12111 2024-09-14T07:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:38.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:38.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:38.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:38.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:38.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:38.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:38.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:38.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:38.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:38.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:38.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:38.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:38.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:38.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:38.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:38.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:38.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:38.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:38.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:38.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:38.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:38.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:38.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:38.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:38.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:38.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:38.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:38.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=12236], [object_number=107, tag_tid=26, entry_tid=12235], [object_number=106, tag_tid=26, entry_tid=12234], [object_number=105, tag_tid=26, entry_tid=12233]]], state=connected] 2024-09-14T07:04:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:38.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:38.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:38.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:38.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:38.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=104, tag_tid=26, entry_tid=12236], [object_number=107, tag_tid=26, entry_tid=12235], [object_number=106, tag_tid=26, entry_tid=12234], [object_number=105, tag_tid=26, entry_tid=12233]]], state=connected] 2024-09-14T07:04:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=12236' '!=' 'object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=104, tag_tid=26, entry_tid=12236' = 'object_number=107, tag_tid=26, entry_tid=12111' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:38.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:38.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:38.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:38.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:38.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:38.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:38.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:38.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:38.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:38.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12236 2024-09-14T07:04:38.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12236 2024-09-14T07:04:38.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:38.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:38.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:39.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:39.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:39.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:39.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:39.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:39.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:39.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:39.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:39.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:39.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=12321], [object_number=104, tag_tid=26, entry_tid=12320], [object_number=107, tag_tid=26, entry_tid=12319], [object_number=106, tag_tid=26, entry_tid=12318]]], state=connected] 2024-09-14T07:04:39.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:39.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:39.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:39.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:39.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:39.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:39.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:39.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:39.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=12321], [object_number=104, tag_tid=26, entry_tid=12320], [object_number=107, tag_tid=26, entry_tid=12319], [object_number=106, tag_tid=26, entry_tid=12318]]], state=connected] 2024-09-14T07:04:39.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=12321' '!=' 'object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=12321' = 'object_number=104, tag_tid=26, entry_tid=12236' 2024-09-14T07:04:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:39.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:39.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:39.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:39.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:39.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12321 2024-09-14T07:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12321 2024-09-14T07:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:39.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:39.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:39.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:39.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:39.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:39.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:39.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:39.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:39.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:39.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:39.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:39.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:39.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=12405], [object_number=104, tag_tid=26, entry_tid=12404], [object_number=107, tag_tid=26, entry_tid=12403], [object_number=106, tag_tid=26, entry_tid=12402]]], state=connected] 2024-09-14T07:04:39.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:39.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:39.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:39.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:39.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=12405], [object_number=104, tag_tid=26, entry_tid=12404], [object_number=107, tag_tid=26, entry_tid=12403], [object_number=106, tag_tid=26, entry_tid=12402]]], state=connected] 2024-09-14T07:04:39.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=12405' '!=' 'object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=12405' = 'object_number=105, tag_tid=26, entry_tid=12321' 2024-09-14T07:04:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:39.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:39.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:39.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:39.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12405 2024-09-14T07:04:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12405 2024-09-14T07:04:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:40.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:40.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:40.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:40.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:40.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:40.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:40.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:40.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:40.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:40.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:40.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:40.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:40.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:40.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:40.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:40.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=12491], [object_number=106, tag_tid=26, entry_tid=12490], [object_number=105, tag_tid=26, entry_tid=12489], [object_number=104, tag_tid=26, entry_tid=12488]]], state=connected] 2024-09-14T07:04:40.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:40.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:40.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:40.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:40.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:40.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=107, tag_tid=26, entry_tid=12491], [object_number=106, tag_tid=26, entry_tid=12490], [object_number=105, tag_tid=26, entry_tid=12489], [object_number=104, tag_tid=26, entry_tid=12488]]], state=connected] 2024-09-14T07:04:40.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=12491' '!=' 'object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=107, tag_tid=26, entry_tid=12491' = 'object_number=105, tag_tid=26, entry_tid=12405' 2024-09-14T07:04:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:40.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:40.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:40.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:40.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12491 2024-09-14T07:04:40.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12491 2024-09-14T07:04:40.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:40.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:40.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:40.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:40.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:40.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:40.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:40.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:40.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:40.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:40.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:40.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:40.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:40.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:40.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=12577], [object_number=104, tag_tid=26, entry_tid=12576], [object_number=107, tag_tid=26, entry_tid=12575], [object_number=106, tag_tid=26, entry_tid=12574]]], state=connected] 2024-09-14T07:04:40.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:40.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:40.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:40.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:40.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=26, entry_tid=12577], [object_number=104, tag_tid=26, entry_tid=12576], [object_number=107, tag_tid=26, entry_tid=12575], [object_number=106, tag_tid=26, entry_tid=12574]]], state=connected] 2024-09-14T07:04:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=26, entry_tid=12577' 2024-09-14T07:04:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=26, entry_tid=12577' 2024-09-14T07:04:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=12577' '!=' 'object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:04:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=105, tag_tid=26, entry_tid=12577' = 'object_number=107, tag_tid=26, entry_tid=12491' 2024-09-14T07:04:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=105, tag_tid=26, entry_tid=12577' 2024-09-14T07:04:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=27 2024-09-14T07:04:40.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=12577' 2024-09-14T07:04:40.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:04:40.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=26 2024-09-14T07:04:40.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:40.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:40.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:04:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=105, tag_tid=26, entry_tid=12577' 2024-09-14T07:04:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:04:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:04:40.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12577 2024-09-14T07:04:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 27 = 26 -a 1 -le 12577 2024-09-14T07:04:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:04:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:04:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:04:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:04:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:04:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:04:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:04:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:04:41.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:04:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:04:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:41.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:04:41.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:04:41.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:04:41.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:04:41.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:41.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:41.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:04:41.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:04:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:41.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:04:41.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:41.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:41.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:41.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:41.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:41.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:41.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:41.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:41.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:41.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:41.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:04:41.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:04:41.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:04:41.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:04:41.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:04:41.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:04:41.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:04:41.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:41.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:04:41.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 26 2024-09-14T07:04:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 26 2024-09-14T07:04:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:04:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=105, tag_tid=27, entry_tid=1], [object_number=104, tag_tid=27, entry_tid=0], [object_number=107, tag_tid=26, entry_tid=12635], [object_number=106, tag_tid=26, entry_tid=12634]]], state=connected] 2024-09-14T07:04:41.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=105, tag_tid=27, entry_tid=1' -a 'object_number=105, tag_tid=27, entry_tid=1' = 'object_number=105, tag_tid=27, entry_tid=1' 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9 2024-09-14T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:04:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:04:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:04:42.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 17 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:04:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:02:35 2024 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:04:04 2024 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:04:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:04:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap8-clean ']' 2024-09-14T07:04:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap8-clean 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8-clean 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:04:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:04:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:44.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:04:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:44.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap9 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:04:45.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:04:45.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:46.287 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:04:46.287+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:04:46.288 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:04:46.291+0000 7fea4eb9a640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:04:46.290 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:04:46.291+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:04:46.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:04:47.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:47.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap9-clean 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap9-clean 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9-clean 2024-09-14T07:04:47.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T07:04:48.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:04:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror test 2024-09-14T07:04:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:04:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:04:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:04:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:04:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=14 2024-09-14T07:04:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:04:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-14T07:04:50.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 20501504 bytes 2024-09-14T07:04:51.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 44865536 bytes 2024-09-14T07:04:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 117402624 bytes 2024-09-14T07:04:55.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 151134208 bytes 2024-09-14T07:04:59.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 244210688 bytes 2024-09-14T07:05:02.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:600 IOs, 284601344 bytes 2024-09-14T07:05:02.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:05:02.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:05:02.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:02.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:05:03.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-14T07:05:03.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:03.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local description_pattern= 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local service_pattern= 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:05:03.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-14T07:05:03.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='test: 2024-09-14T07:05:03.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: global_id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:05:03.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+replaying 2024-09-14T07:05:03.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: replaying, {"bytes_per_second":3068391.73,"entries_behind_primary":145,"entries_per_second":189.63,"non_primary_position":{"entry_tid":12491,"object_number":107,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":105,"tag_tid":27},"seconds_until_synced":0} 2024-09-14T07:05:03.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: service: mirror.0 on smithi169 2024-09-14T07:05:03.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:04:40 2024-09-14T07:05:03.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: peer_sites: 2024-09-14T07:05:03.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: name: cluster2 2024-09-14T07:05:03.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: state: up+stopped 2024-09-14T07:05:03.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: description: local image is primary 2024-09-14T07:05:03.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: last_update: 2024-09-14 07:04:40' 2024-09-14T07:05:03.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ state: .*up+replaying' 2024-09-14T07:05:03.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: state: up+replaying 2024-09-14T07:05:03.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^ description: .*' 2024-09-14T07:05:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: description: replaying, {"bytes_per_second":3068391.73,"entries_behind_primary":145,"entries_per_second":189.63,"non_primary_position":{"entry_tid":12491,"object_number":107,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":105,"tag_tid":27},"seconds_until_synced":0} 2024-09-14T07:05:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:05:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:05:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep '^up+' 2024-09-14T07:05:03.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:05:03.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep 'service: *mirror..* on ' 2024-09-14T07:05:03.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: service: mirror.0 on smithi169 2024-09-14T07:05:03.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update 2024-09-14T07:05:03.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-14T07:05:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:04:40' 2024-09-14T07:05:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-14 07:04:40' 2024-09-14T07:05:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:03.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state_pattern=up+replaying 2024-09-14T07:05:03.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'prev_last_update=2024-09-14 07:04:40' 2024-09-14T07:05:03.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status 2024-09-14T07:05:03.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ CEPH_ARGS= 2024-09-14T07:05:03.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-14T07:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status='OKOKOK142104261mirror.0smithi16919.3.0-4775-g7f24e589falseOK44074410mirror.2smithi16919.3.0-4775-g7f24e589trueOK44794482mirror.3smithi16919.3.0-4775-g7f24e589falseOK45254528mirror.1smithi16919.3.0-4775-g7f24e589falseOKtesta5d62446-6732-43cb-bbad-b5dfd05689bdup+replayingreplaying, {"bytes_per_second":3068391.73,"entries_behind_primary":145,"entries_per_second":189.63,"non_primary_position":{"entry_tid":12491,"object_number":107,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":105,"tag_tid":27},"seconds_until_synced":0}42104261mirror.0smithi1692024-09-14 07:04:40cluster2e9027ad0-e174-4559-b61e-a827147f3ce9up+stoppedlocal image is primary2024-09-14 07:04:40' 2024-09-14T07:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_update state 2024-09-14T07:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-14T07:05:03.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_update='2024-09-14 07:04:40' 2024-09-14T07:05:03.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-14T07:05:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=up+replaying 2024-09-14T07:05:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo up+replaying 2024-09-14T07:05:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep up+replaying 2024-09-14T07:05:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:up+replaying 2024-09-14T07:05:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap10 2024-09-14T07:05:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap10 2024-09-14T07:05:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:05:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:03.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:03.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10 2024-09-14T07:05:03.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:05:34.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T07:05:34.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:34.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:34.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:34.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T07:05:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:05:35.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:35.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:05:35.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:05:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:05:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:05:35.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:05:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:35.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:35.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:35.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:35.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:05:36.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:05:36.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:05:36.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:05:36.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:05:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:36.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:36.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:36.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:36.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:36.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:36.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:36.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:36.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:36.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:36.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:36.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:36.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:36.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:36.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:36.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:36.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:36.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:36.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:36.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=8227], [object_number=114, tag_tid=29, entry_tid=8226], [object_number=113, tag_tid=29, entry_tid=8225], [object_number=112, tag_tid=29, entry_tid=8224]]], state=connected] 2024-09-14T07:05:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:36.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:36.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:36.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:36.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=8227], [object_number=114, tag_tid=29, entry_tid=8226], [object_number=113, tag_tid=29, entry_tid=8225], [object_number=112, tag_tid=29, entry_tid=8224]]], state=connected] 2024-09-14T07:05:36.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=8227' '!=' '' 2024-09-14T07:05:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=8227' = '' 2024-09-14T07:05:36.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:36.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:36.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:36.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:36.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:36.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:36.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:36.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:36.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:36.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:36.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:36.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8227 2024-09-14T07:05:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8227 2024-09-14T07:05:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:36.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:36.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:36.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:36.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:36.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:36.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:36.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:36.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:36.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:36.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:36.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=8328], [object_number=115, tag_tid=29, entry_tid=8327], [object_number=114, tag_tid=29, entry_tid=8326], [object_number=113, tag_tid=29, entry_tid=8325]]], state=connected] 2024-09-14T07:05:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:36.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:36.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:36.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:36.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:36.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:36.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:37.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:37.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:37.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:37.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=8328], [object_number=115, tag_tid=29, entry_tid=8327], [object_number=114, tag_tid=29, entry_tid=8326], [object_number=113, tag_tid=29, entry_tid=8325]]], state=connected] 2024-09-14T07:05:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=8328' '!=' 'object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=8328' = 'object_number=115, tag_tid=29, entry_tid=8227' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:37.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:37.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:37.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8328 2024-09-14T07:05:37.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8328 2024-09-14T07:05:37.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:37.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:37.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:37.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:37.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:37.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:37.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:37.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:37.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:37.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:37.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:37.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:37.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:37.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:37.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:37.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:37.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:37.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=113, tag_tid=29, entry_tid=8433], [object_number=112, tag_tid=29, entry_tid=8432], [object_number=115, tag_tid=29, entry_tid=8431], [object_number=114, tag_tid=29, entry_tid=8430]]], state=connected] 2024-09-14T07:05:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:37.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:37.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:37.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:37.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:37.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=113, tag_tid=29, entry_tid=8433], [object_number=112, tag_tid=29, entry_tid=8432], [object_number=115, tag_tid=29, entry_tid=8431], [object_number=114, tag_tid=29, entry_tid=8430]]], state=connected] 2024-09-14T07:05:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=113, tag_tid=29, entry_tid=8433' '!=' 'object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=113, tag_tid=29, entry_tid=8433' = 'object_number=112, tag_tid=29, entry_tid=8328' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:37.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:37.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:37.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:37.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8433 2024-09-14T07:05:37.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8433 2024-09-14T07:05:37.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:37.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:37.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:37.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:37.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:37.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:37.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:37.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:37.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:37.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:37.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:37.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:37.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:37.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:37.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:37.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:37.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:37.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:37.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:37.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=8534], [object_number=113, tag_tid=29, entry_tid=8533], [object_number=112, tag_tid=29, entry_tid=8532], [object_number=115, tag_tid=29, entry_tid=8531]]], state=connected] 2024-09-14T07:05:37.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:37.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:37.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=8534], [object_number=113, tag_tid=29, entry_tid=8533], [object_number=112, tag_tid=29, entry_tid=8532], [object_number=115, tag_tid=29, entry_tid=8531]]], state=connected] 2024-09-14T07:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=8534' '!=' 'object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:38.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=8534' = 'object_number=113, tag_tid=29, entry_tid=8433' 2024-09-14T07:05:38.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:38.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:38.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:38.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:38.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:38.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:38.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:38.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8534 2024-09-14T07:05:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8534 2024-09-14T07:05:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:38.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:38.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:38.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:38.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:38.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:38.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:38.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:38.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:38.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:38.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:38.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:38.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:38.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=113, tag_tid=29, entry_tid=8645], [object_number=112, tag_tid=29, entry_tid=8644], [object_number=115, tag_tid=29, entry_tid=8643], [object_number=114, tag_tid=29, entry_tid=8642]]], state=connected] 2024-09-14T07:05:38.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:38.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:38.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:38.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:38.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=113, tag_tid=29, entry_tid=8645], [object_number=112, tag_tid=29, entry_tid=8644], [object_number=115, tag_tid=29, entry_tid=8643], [object_number=114, tag_tid=29, entry_tid=8642]]], state=connected] 2024-09-14T07:05:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=113, tag_tid=29, entry_tid=8645' '!=' 'object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=113, tag_tid=29, entry_tid=8645' = 'object_number=114, tag_tid=29, entry_tid=8534' 2024-09-14T07:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:38.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:38.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:38.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:38.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:38.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:38.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:38.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8645 2024-09-14T07:05:38.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8645 2024-09-14T07:05:38.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:38.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:38.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:38.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:38.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:38.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:38.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:38.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:38.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:38.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:38.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:38.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:38.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:38.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:38.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:38.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:38.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:38.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=8751], [object_number=114, tag_tid=29, entry_tid=8750], [object_number=113, tag_tid=29, entry_tid=8749], [object_number=112, tag_tid=29, entry_tid=8748]]], state=connected] 2024-09-14T07:05:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=8751], [object_number=114, tag_tid=29, entry_tid=8750], [object_number=113, tag_tid=29, entry_tid=8749], [object_number=112, tag_tid=29, entry_tid=8748]]], state=connected] 2024-09-14T07:05:39.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=8751' '!=' 'object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=8751' = 'object_number=113, tag_tid=29, entry_tid=8645' 2024-09-14T07:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:39.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:39.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8751 2024-09-14T07:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8751 2024-09-14T07:05:39.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:39.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:39.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:39.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:39.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:39.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:39.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:39.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:39.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:39.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:39.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:39.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:39.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:39.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:39.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:39.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:39.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:39.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:39.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:39.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:39.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=8846], [object_number=113, tag_tid=29, entry_tid=8845], [object_number=112, tag_tid=29, entry_tid=8844], [object_number=115, tag_tid=29, entry_tid=8843]]], state=connected] 2024-09-14T07:05:39.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:39.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:39.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:39.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:39.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:39.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:39.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:39.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=8846], [object_number=113, tag_tid=29, entry_tid=8845], [object_number=112, tag_tid=29, entry_tid=8844], [object_number=115, tag_tid=29, entry_tid=8843]]], state=connected] 2024-09-14T07:05:39.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=8846' '!=' 'object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=8846' = 'object_number=115, tag_tid=29, entry_tid=8751' 2024-09-14T07:05:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:39.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:39.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:39.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8846 2024-09-14T07:05:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8846 2024-09-14T07:05:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:39.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:39.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:39.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:39.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=8948], [object_number=115, tag_tid=29, entry_tid=8947], [object_number=114, tag_tid=29, entry_tid=8946], [object_number=113, tag_tid=29, entry_tid=8945]]], state=connected] 2024-09-14T07:05:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:39.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=8966], [object_number=113, tag_tid=29, entry_tid=8965], [object_number=112, tag_tid=29, entry_tid=8964], [object_number=115, tag_tid=29, entry_tid=8963]]], state=connected] 2024-09-14T07:05:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=8966' '!=' 'object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=8966' = 'object_number=114, tag_tid=29, entry_tid=8846' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:39.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:39.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:39.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8966 2024-09-14T07:05:40.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 8966 2024-09-14T07:05:40.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:40.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:40.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:40.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:40.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:40.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:40.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:40.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:40.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:40.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:40.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:40.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:40.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:40.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:40.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:40.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:40.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=113, tag_tid=29, entry_tid=9037], [object_number=112, tag_tid=29, entry_tid=9036], [object_number=115, tag_tid=29, entry_tid=9035], [object_number=114, tag_tid=29, entry_tid=9034]]], state=connected] 2024-09-14T07:05:40.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:40.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=113, tag_tid=29, entry_tid=9037], [object_number=112, tag_tid=29, entry_tid=9036], [object_number=115, tag_tid=29, entry_tid=9035], [object_number=114, tag_tid=29, entry_tid=9034]]], state=connected] 2024-09-14T07:05:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=113, tag_tid=29, entry_tid=9037' '!=' 'object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=113, tag_tid=29, entry_tid=9037' = 'object_number=114, tag_tid=29, entry_tid=8966' 2024-09-14T07:05:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:40.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9037 2024-09-14T07:05:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9037 2024-09-14T07:05:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:40.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:40.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:40.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:40.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:40.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:40.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:40.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:40.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:40.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:40.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:40.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:40.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:40.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=9135], [object_number=114, tag_tid=29, entry_tid=9134], [object_number=113, tag_tid=29, entry_tid=9133], [object_number=112, tag_tid=29, entry_tid=9132]]], state=connected] 2024-09-14T07:05:40.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:40.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:40.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:40.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:40.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:40.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:40.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=9135], [object_number=114, tag_tid=29, entry_tid=9134], [object_number=113, tag_tid=29, entry_tid=9133], [object_number=112, tag_tid=29, entry_tid=9132]]], state=connected] 2024-09-14T07:05:40.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:40.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:40.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:40.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=9135' '!=' 'object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:40.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=9135' = 'object_number=113, tag_tid=29, entry_tid=9037' 2024-09-14T07:05:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9135 2024-09-14T07:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9135 2024-09-14T07:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:41.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:41.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:41.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:41.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:41.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:41.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:41.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=9230], [object_number=113, tag_tid=29, entry_tid=9229], [object_number=112, tag_tid=29, entry_tid=9228], [object_number=115, tag_tid=29, entry_tid=9227]]], state=connected] 2024-09-14T07:05:41.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:41.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:41.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=114, tag_tid=29, entry_tid=9230], [object_number=113, tag_tid=29, entry_tid=9229], [object_number=112, tag_tid=29, entry_tid=9228], [object_number=115, tag_tid=29, entry_tid=9227]]], state=connected] 2024-09-14T07:05:41.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=9230' '!=' 'object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=114, tag_tid=29, entry_tid=9230' = 'object_number=115, tag_tid=29, entry_tid=9135' 2024-09-14T07:05:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:41.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:41.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:41.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:41.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:41.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:41.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9230 2024-09-14T07:05:41.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9230 2024-09-14T07:05:41.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:41.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:41.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:41.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:41.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:41.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:41.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:41.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:41.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:41.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:41.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:41.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:41.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:41.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:41.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:41.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:41.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:41.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9324], [object_number=115, tag_tid=29, entry_tid=9323], [object_number=114, tag_tid=29, entry_tid=9322], [object_number=113, tag_tid=29, entry_tid=9321]]], state=connected] 2024-09-14T07:05:41.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:41.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:41.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:41.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:41.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:41.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:41.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:41.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:41.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9324], [object_number=115, tag_tid=29, entry_tid=9323], [object_number=114, tag_tid=29, entry_tid=9322], [object_number=113, tag_tid=29, entry_tid=9321]]], state=connected] 2024-09-14T07:05:41.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9324' '!=' 'object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9324' = 'object_number=114, tag_tid=29, entry_tid=9230' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:41.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:41.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:41.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:41.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:41.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:41.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9324 2024-09-14T07:05:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9324 2024-09-14T07:05:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:41.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:42.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:42.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:42.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:42.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:42.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:42.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:42.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:42.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:42.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:42.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9412], [object_number=115, tag_tid=29, entry_tid=9411], [object_number=114, tag_tid=29, entry_tid=9410], [object_number=113, tag_tid=29, entry_tid=9409]]], state=connected] 2024-09-14T07:05:42.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:42.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:42.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:42.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9412], [object_number=115, tag_tid=29, entry_tid=9411], [object_number=114, tag_tid=29, entry_tid=9410], [object_number=113, tag_tid=29, entry_tid=9409]]], state=connected] 2024-09-14T07:05:42.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9412' '!=' 'object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:42.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:42.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9412' = 'object_number=112, tag_tid=29, entry_tid=9324' 2024-09-14T07:05:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:42.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:42.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9412 2024-09-14T07:05:42.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9412 2024-09-14T07:05:42.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:42.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:42.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:42.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:42.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:42.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:42.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:42.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:42.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:42.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:42.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=9495], [object_number=114, tag_tid=29, entry_tid=9494], [object_number=113, tag_tid=29, entry_tid=9493], [object_number=112, tag_tid=29, entry_tid=9492]]], state=connected] 2024-09-14T07:05:42.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:42.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:42.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=9495], [object_number=114, tag_tid=29, entry_tid=9494], [object_number=113, tag_tid=29, entry_tid=9493], [object_number=112, tag_tid=29, entry_tid=9492]]], state=connected] 2024-09-14T07:05:42.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=9495' '!=' 'object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=9495' = 'object_number=112, tag_tid=29, entry_tid=9412' 2024-09-14T07:05:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:42.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:42.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:42.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:42.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:42.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:42.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:42.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:42.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:42.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9495 2024-09-14T07:05:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9495 2024-09-14T07:05:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:43.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:43.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:43.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:43.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:43.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:43.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:43.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:43.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:43.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:43.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:43.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:43.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:43.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:43.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:43.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:43.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:43.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:43.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:43.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=9579], [object_number=114, tag_tid=29, entry_tid=9578], [object_number=113, tag_tid=29, entry_tid=9577], [object_number=112, tag_tid=29, entry_tid=9576]]], state=connected] 2024-09-14T07:05:43.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:43.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:43.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:43.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:43.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:43.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:43.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:43.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:43.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=115, tag_tid=29, entry_tid=9579], [object_number=114, tag_tid=29, entry_tid=9578], [object_number=113, tag_tid=29, entry_tid=9577], [object_number=112, tag_tid=29, entry_tid=9576]]], state=connected] 2024-09-14T07:05:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=9579' '!=' 'object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=115, tag_tid=29, entry_tid=9579' = 'object_number=115, tag_tid=29, entry_tid=9495' 2024-09-14T07:05:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:43.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:43.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:43.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:43.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:43.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:43.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:43.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:43.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:43.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:43.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:43.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9579 2024-09-14T07:05:43.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9579 2024-09-14T07:05:43.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:43.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:43.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:43.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:43.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:43.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:43.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:43.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:43.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:43.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:43.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:43.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:43.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:43.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:43.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:43.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:44.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:44.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:44.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:44.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:44.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:44.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:44.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:44.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:44.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9708], [object_number=115, tag_tid=29, entry_tid=9707], [object_number=114, tag_tid=29, entry_tid=9706], [object_number=113, tag_tid=29, entry_tid=9705]]], state=connected] 2024-09-14T07:05:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:44.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:44.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9708], [object_number=115, tag_tid=29, entry_tid=9707], [object_number=114, tag_tid=29, entry_tid=9706], [object_number=113, tag_tid=29, entry_tid=9705]]], state=connected] 2024-09-14T07:05:44.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9708' '!=' 'object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9708' = 'object_number=115, tag_tid=29, entry_tid=9579' 2024-09-14T07:05:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:44.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:44.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9708 2024-09-14T07:05:44.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9708 2024-09-14T07:05:44.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:44.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:44.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:44.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:44.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:44.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:44.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:44.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:44.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:44.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:44.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:44.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9800], [object_number=115, tag_tid=29, entry_tid=9799], [object_number=114, tag_tid=29, entry_tid=9798], [object_number=113, tag_tid=29, entry_tid=9797]]], state=connected] 2024-09-14T07:05:44.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 28 2024-09-14T07:05:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=112, tag_tid=29, entry_tid=9800], [object_number=115, tag_tid=29, entry_tid=9799], [object_number=114, tag_tid=29, entry_tid=9798], [object_number=113, tag_tid=29, entry_tid=9797]]], state=connected] 2024-09-14T07:05:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9800' '!=' 'object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=112, tag_tid=29, entry_tid=9800' = 'object_number=112, tag_tid=29, entry_tid=9708' 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:44.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:44.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:44.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:44.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:44.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:44.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:44.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9800 2024-09-14T07:05:44.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9800 2024-09-14T07:05:44.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:44.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:44.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:44.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:44.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:44.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:44.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:44.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:44.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:45.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:45.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:45.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:45.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:45.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=9867], [object_number=118, tag_tid=29, entry_tid=9866], [object_number=117, tag_tid=29, entry_tid=9865], [object_number=116, tag_tid=29, entry_tid=9864]]], state=connected] 2024-09-14T07:05:45.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:45.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:45.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:45.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:45.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:45.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=9867], [object_number=118, tag_tid=29, entry_tid=9866], [object_number=117, tag_tid=29, entry_tid=9865], [object_number=116, tag_tid=29, entry_tid=9864]]], state=connected] 2024-09-14T07:05:45.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=9867' '!=' 'object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=9867' = 'object_number=112, tag_tid=29, entry_tid=9800' 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:45.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:45.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9867 2024-09-14T07:05:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9867 2024-09-14T07:05:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:45.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:45.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:45.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:45.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:45.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:45.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:45.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:45.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:45.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:45.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:45.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:45.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:45.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:45.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:45.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:45.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:45.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:45.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:45.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:45.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:45.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=9961], [object_number=116, tag_tid=29, entry_tid=9960], [object_number=119, tag_tid=29, entry_tid=9959], [object_number=118, tag_tid=29, entry_tid=9958]]], state=connected] 2024-09-14T07:05:45.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:45.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=9961], [object_number=116, tag_tid=29, entry_tid=9960], [object_number=119, tag_tid=29, entry_tid=9959], [object_number=118, tag_tid=29, entry_tid=9958]]], state=connected] 2024-09-14T07:05:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=9961' '!=' 'object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=9961' = 'object_number=119, tag_tid=29, entry_tid=9867' 2024-09-14T07:05:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:45.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:45.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:45.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:45.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:45.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:45.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9961 2024-09-14T07:05:45.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 9961 2024-09-14T07:05:45.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:45.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:45.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:45.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:45.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:45.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:45.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:45.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:45.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:45.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:45.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:45.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:45.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:45.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:45.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:45.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:45.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:45.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:46.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:46.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:46.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=10058], [object_number=117, tag_tid=29, entry_tid=10057], [object_number=116, tag_tid=29, entry_tid=10056], [object_number=119, tag_tid=29, entry_tid=10055]]], state=connected] 2024-09-14T07:05:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:46.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:46.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:46.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:46.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:46.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:46.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:46.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:46.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=10058], [object_number=117, tag_tid=29, entry_tid=10057], [object_number=116, tag_tid=29, entry_tid=10056], [object_number=119, tag_tid=29, entry_tid=10055]]], state=connected] 2024-09-14T07:05:46.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=10058' '!=' 'object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=10058' = 'object_number=117, tag_tid=29, entry_tid=9961' 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:46.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:46.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:46.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:46.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:46.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10058 2024-09-14T07:05:46.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10058 2024-09-14T07:05:46.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:46.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:46.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:46.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:46.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:46.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:46.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:46.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:46.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:46.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:46.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:46.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:46.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:46.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:46.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:46.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:46.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:46.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:46.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:46.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:46.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:46.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:46.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:46.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:46.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:46.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:46.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:46.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:46.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:46.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:46.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:46.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:46.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:46.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:46.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:46.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:46.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:46.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10159], [object_number=118, tag_tid=29, entry_tid=10158], [object_number=117, tag_tid=29, entry_tid=10157], [object_number=116, tag_tid=29, entry_tid=10156]]], state=connected] 2024-09-14T07:05:47.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:47.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:47.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:47.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:47.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:47.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:47.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:47.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:47.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:47.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:47.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:47.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10159], [object_number=118, tag_tid=29, entry_tid=10158], [object_number=117, tag_tid=29, entry_tid=10157], [object_number=116, tag_tid=29, entry_tid=10156]]], state=connected] 2024-09-14T07:05:47.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10159' '!=' 'object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10159' = 'object_number=118, tag_tid=29, entry_tid=10058' 2024-09-14T07:05:47.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:47.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:47.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:47.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:47.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:47.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:47.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10159 2024-09-14T07:05:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10159 2024-09-14T07:05:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:47.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:47.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:47.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:47.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:47.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:47.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:47.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:47.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:47.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:47.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:47.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=10409], [object_number=116, tag_tid=29, entry_tid=10408], [object_number=119, tag_tid=29, entry_tid=10407], [object_number=118, tag_tid=29, entry_tid=10406]]], state=connected] 2024-09-14T07:05:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:47.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:47.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:47.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:47.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:47.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:47.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=10409], [object_number=116, tag_tid=29, entry_tid=10408], [object_number=119, tag_tid=29, entry_tid=10407], [object_number=118, tag_tid=29, entry_tid=10406]]], state=connected] 2024-09-14T07:05:47.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:47.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:47.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:47.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=10409' '!=' 'object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:47.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=10409' = 'object_number=119, tag_tid=29, entry_tid=10159' 2024-09-14T07:05:47.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:47.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:47.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:47.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:47.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:47.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:47.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10409 2024-09-14T07:05:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10409 2024-09-14T07:05:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:47.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:48.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:48.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:48.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:48.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:48.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:48.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:48.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:48.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:48.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:48.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:48.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:48.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:48.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:48.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:48.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:48.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:48.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:48.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10503], [object_number=118, tag_tid=29, entry_tid=10502], [object_number=117, tag_tid=29, entry_tid=10501], [object_number=116, tag_tid=29, entry_tid=10500]]], state=connected] 2024-09-14T07:05:48.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10503], [object_number=118, tag_tid=29, entry_tid=10502], [object_number=117, tag_tid=29, entry_tid=10501], [object_number=116, tag_tid=29, entry_tid=10500]]], state=connected] 2024-09-14T07:05:48.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10503' '!=' 'object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10503' = 'object_number=117, tag_tid=29, entry_tid=10409' 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10503 2024-09-14T07:05:48.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10503 2024-09-14T07:05:48.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:48.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:48.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:48.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:48.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:48.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:48.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:48.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:48.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:48.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:48.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:48.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:48.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:48.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:48.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:48.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:48.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:48.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:48.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:48.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:48.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:48.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:48.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:48.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:48.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10599], [object_number=118, tag_tid=29, entry_tid=10598], [object_number=117, tag_tid=29, entry_tid=10597], [object_number=116, tag_tid=29, entry_tid=10596]]], state=connected] 2024-09-14T07:05:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:48.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:48.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:48.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:48.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:48.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10599], [object_number=118, tag_tid=29, entry_tid=10598], [object_number=117, tag_tid=29, entry_tid=10597], [object_number=116, tag_tid=29, entry_tid=10596]]], state=connected] 2024-09-14T07:05:48.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10599' '!=' 'object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10599' = 'object_number=119, tag_tid=29, entry_tid=10503' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:48.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:48.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:48.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:48.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10599 2024-09-14T07:05:48.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10599 2024-09-14T07:05:48.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:48.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:48.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:49.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:49.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:49.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:49.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:49.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:49.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:49.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=10704], [object_number=119, tag_tid=29, entry_tid=10703], [object_number=118, tag_tid=29, entry_tid=10702], [object_number=117, tag_tid=29, entry_tid=10701]]], state=connected] 2024-09-14T07:05:49.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:49.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:49.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:49.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:49.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:49.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:49.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=10704], [object_number=119, tag_tid=29, entry_tid=10703], [object_number=118, tag_tid=29, entry_tid=10702], [object_number=117, tag_tid=29, entry_tid=10701]]], state=connected] 2024-09-14T07:05:49.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=10704' '!=' 'object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=10704' = 'object_number=119, tag_tid=29, entry_tid=10599' 2024-09-14T07:05:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:49.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:49.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:49.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:49.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:49.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10704 2024-09-14T07:05:49.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10704 2024-09-14T07:05:49.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:49.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:49.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:49.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:49.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:49.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:49.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:49.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:49.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:49.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:49.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:49.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:49.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:49.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:49.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=10792], [object_number=119, tag_tid=29, entry_tid=10791], [object_number=118, tag_tid=29, entry_tid=10790], [object_number=117, tag_tid=29, entry_tid=10789]]], state=connected] 2024-09-14T07:05:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:49.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:49.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:49.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:49.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:49.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:49.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:49.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=10792], [object_number=119, tag_tid=29, entry_tid=10791], [object_number=118, tag_tid=29, entry_tid=10790], [object_number=117, tag_tid=29, entry_tid=10789]]], state=connected] 2024-09-14T07:05:49.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=10792' '!=' 'object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=10792' = 'object_number=116, tag_tid=29, entry_tid=10704' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:49.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:49.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:49.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:49.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:49.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:49.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:49.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10792 2024-09-14T07:05:49.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10792 2024-09-14T07:05:49.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:49.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:49.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:49.843 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9715326359868744) > weight (0.25) 2024-09-14T07:05:49.843 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3903132681935416) > weight (0.25) 2024-09-14T07:05:49.843 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:05:49.844 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:05:49.844 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T07:05:49.845 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:05:49.844+0000 7f736eecc640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 59436) UID: 1000 2024-09-14T07:05:50.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:50.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:50.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:50.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:50.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:50.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:50.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:50.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:50.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:50.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:50.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:50.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:50.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:50.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:50.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:50.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:50.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10887], [object_number=118, tag_tid=29, entry_tid=10886], [object_number=117, tag_tid=29, entry_tid=10885], [object_number=116, tag_tid=29, entry_tid=10884]]], state=connected] 2024-09-14T07:05:50.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:50.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:50.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:50.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:50.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:50.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10887], [object_number=118, tag_tid=29, entry_tid=10886], [object_number=117, tag_tid=29, entry_tid=10885], [object_number=116, tag_tid=29, entry_tid=10884]]], state=connected] 2024-09-14T07:05:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10887' '!=' 'object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10887' = 'object_number=116, tag_tid=29, entry_tid=10792' 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:50.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:50.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:50.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:50.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:50.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:50.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10887 2024-09-14T07:05:50.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10887 2024-09-14T07:05:50.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:50.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:50.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:50.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:50.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:50.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:50.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:50.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:50.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:50.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:50.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:50.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:50.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:50.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:50.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:50.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:50.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:50.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:50.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:05:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:05:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:05:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:05:50.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:50.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:50.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:50.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10983], [object_number=118, tag_tid=29, entry_tid=10982], [object_number=117, tag_tid=29, entry_tid=10981], [object_number=116, tag_tid=29, entry_tid=10980]]], state=connected] 2024-09-14T07:05:50.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:05:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:05:50.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:05:50.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.743 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:05:50.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:05:50.743 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:05:50.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:05:50.744+0000 7fb490243640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 30403) UID: 1000 2024-09-14T07:05:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:05:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:05:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:05:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:05:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=10983], [object_number=118, tag_tid=29, entry_tid=10982], [object_number=117, tag_tid=29, entry_tid=10981], [object_number=116, tag_tid=29, entry_tid=10980]]], state=connected] 2024-09-14T07:05:50.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:05:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:05:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10983' '!=' 'object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=10983' = 'object_number=119, tag_tid=29, entry_tid=10887' 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:05:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:05:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:05:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:05:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:05:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:05:50.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:05:50.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:05:50.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:05:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10983 2024-09-14T07:05:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 10983 2024-09-14T07:05:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:05:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:05:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:05:50.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:05:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:50.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:05:50.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:50.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:50.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:51.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:51.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:51.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:51.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:05:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:05:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:05:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:05:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:05:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:05:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:05:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:05:51.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:05:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:05:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:51.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:51.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:51.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:05:51.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:05:51.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:51.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:05:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:51.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:05:51.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:05:51.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:51.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:51.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:05:51.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:05:51.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:51.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:51.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:51.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:51.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:51.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:51.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:51.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:05:51.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:05:51.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:05:51.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:51.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:05:51.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:51.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:51.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:05:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:05:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:51.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:05:51.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:05:51.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:51.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:05:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:05:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:51.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:51.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:05:51.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:51.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:51.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:51.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:51.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:05:52.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:52.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:52.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:52.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:52.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:52.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:52.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:52.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:52.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:52.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:05:52.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:52.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:05:52.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:52.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:05:52.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:05:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:05:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:05:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:05:52.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:52.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:52.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:05:52.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:05:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:05:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:05:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:05:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:05:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:05:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:52.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:05:52.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:52.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:05:52.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:05:52.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:05:52.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:05:52.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:52.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:52.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:52.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:05:53.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:53.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:53.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:05:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:55.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:55.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:55.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:55.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:55.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:55.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:55.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:05:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:05:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:05:55.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:55.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:05:55.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:05:55.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:05:55.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:55.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:55.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:55.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:05:55.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:55.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:55.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:55.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:55.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:05:55.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:05:55.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:05:55.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:55.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:55.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:55.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:05:55.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:05:55.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:05:55.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:55.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:05:55.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:55.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:55.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:05:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:05:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:05:55.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:05:55.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:55.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:55.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:55.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:05:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:05:55.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 4 2024-09-14T07:05:55.845 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:05:55.845 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:05:55.846 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:05:55.846 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:05:55.846 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:05:55.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:05:55.849 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-09-14T07:05:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:05:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:05:59.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:05:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:05:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:05:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:05:59.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:59.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:59.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:05:59.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:05:59.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:05:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:05:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:05:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:05:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:05:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:05:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:05:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:59.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:59.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:05:59.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:05:59.744 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:05:59.744 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:05:59.745 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:05:59.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:05:59.745 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:05:59.747 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:05:59.748 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-09-14T07:05:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:59.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:05:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:05:59.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:59.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:59.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:05:59.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:05:59.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:05:59.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:05:59.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:05:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:59.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:05:59.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:06:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:06:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:08.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:08.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:08.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:08.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:08.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:08.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:08.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:08.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:08.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:08.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:08.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:08.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:08.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:08.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:08.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:06:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:06:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:06:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:06:08.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:06:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:06:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:06:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:06:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:06:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:06:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:06:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:06:16.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:16.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:16.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:16.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:16.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:16.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:16.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:16.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:16.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:16.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:16.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:16.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:17.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:17.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:17.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:17.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:17.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:06:17.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:06:17.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:06:17.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:06:17.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:06:17.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:06:17.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:06:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:06:25.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:25.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:25.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:25.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:25.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:25.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:25.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:25.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:25.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:25.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:25.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:25.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:25.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:25.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:25.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:25.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:25.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:25.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:25.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:25.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:25.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:25.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:25.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:25.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=11549], [object_number=116, tag_tid=29, entry_tid=11548], [object_number=119, tag_tid=29, entry_tid=11547], [object_number=118, tag_tid=29, entry_tid=11546]]], state=connected] 2024-09-14T07:06:25.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:25.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:25.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:25.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:25.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:25.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:25.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:25.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=11549], [object_number=116, tag_tid=29, entry_tid=11548], [object_number=119, tag_tid=29, entry_tid=11547], [object_number=118, tag_tid=29, entry_tid=11546]]], state=connected] 2024-09-14T07:06:25.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:25.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:25.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:25.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=11549' '!=' 'object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:06:25.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=11549' = 'object_number=119, tag_tid=29, entry_tid=10983' 2024-09-14T07:06:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:25.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:25.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:25.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11549 2024-09-14T07:06:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 11549 2024-09-14T07:06:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:25.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:25.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:25.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:25.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:25.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:25.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:25.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:25.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:25.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:25.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:25.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:26.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:26.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:26.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:26.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:26.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:26.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:26.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:26.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:26.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:26.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:26.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:26.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:26.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:26.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:26.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:26.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:26.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:26.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:26.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:26.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=11687], [object_number=118, tag_tid=29, entry_tid=11686], [object_number=117, tag_tid=29, entry_tid=11685], [object_number=116, tag_tid=29, entry_tid=11684]]], state=connected] 2024-09-14T07:06:26.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:26.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:26.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:26.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:26.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:26.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=11687], [object_number=118, tag_tid=29, entry_tid=11686], [object_number=117, tag_tid=29, entry_tid=11685], [object_number=116, tag_tid=29, entry_tid=11684]]], state=connected] 2024-09-14T07:06:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:26.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:26.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:26.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=11687' '!=' 'object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:26.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:26.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=11687' = 'object_number=117, tag_tid=29, entry_tid=11549' 2024-09-14T07:06:26.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:26.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:26.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:26.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:26.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:26.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:26.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11687 2024-09-14T07:06:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 11687 2024-09-14T07:06:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:26.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:26.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:26.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:26.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:26.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:26.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:26.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:26.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:26.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:26.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:26.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:26.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:26.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:26.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:26.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:26.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:26.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:26.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:26.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:26.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:26.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:26.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:26.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:26.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:26.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:26.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:26.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:26.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:26.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:26.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:27.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:27.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:27.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:27.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:27.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:27.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:27.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:27.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=11817], [object_number=116, tag_tid=29, entry_tid=11816], [object_number=119, tag_tid=29, entry_tid=11815], [object_number=118, tag_tid=29, entry_tid=11814]]], state=connected] 2024-09-14T07:06:27.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:27.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=11817], [object_number=116, tag_tid=29, entry_tid=11816], [object_number=119, tag_tid=29, entry_tid=11815], [object_number=118, tag_tid=29, entry_tid=11814]]], state=connected] 2024-09-14T07:06:27.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=11817' '!=' 'object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=11817' = 'object_number=119, tag_tid=29, entry_tid=11687' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:27.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:27.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:27.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11817 2024-09-14T07:06:27.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 11817 2024-09-14T07:06:27.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:27.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:27.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:27.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:27.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:27.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:27.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:27.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:27.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:27.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:27.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:27.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:27.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:27.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:27.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:27.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:27.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:27.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:27.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:27.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:27.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:27.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:27.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:27.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:27.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:27.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:27.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:27.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:27.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:27.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=11942], [object_number=117, tag_tid=29, entry_tid=11941], [object_number=116, tag_tid=29, entry_tid=11940], [object_number=119, tag_tid=29, entry_tid=11939]]], state=connected] 2024-09-14T07:06:27.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:27.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:27.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:27.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:27.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:27.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:27.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:27.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:27.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=11942], [object_number=117, tag_tid=29, entry_tid=11941], [object_number=116, tag_tid=29, entry_tid=11940], [object_number=119, tag_tid=29, entry_tid=11939]]], state=connected] 2024-09-14T07:06:27.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=11942' '!=' 'object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=11942' = 'object_number=117, tag_tid=29, entry_tid=11817' 2024-09-14T07:06:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:27.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:27.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:27.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:27.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:27.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:27.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:27.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:27.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:27.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:27.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11942 2024-09-14T07:06:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 11942 2024-09-14T07:06:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:28.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:28.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:28.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:28.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:28.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:28.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:28.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:28.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:28.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:28.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:28.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:28.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:28.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:28.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:28.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:28.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:28.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:28.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:28.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:28.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:28.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:28.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:28.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:28.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:28.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:28.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:28.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:28.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:28.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:28.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:28.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:28.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:28.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:28.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:28.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:28.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:28.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:28.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:28.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:28.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:28.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:28.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=12075], [object_number=118, tag_tid=29, entry_tid=12074], [object_number=117, tag_tid=29, entry_tid=12073], [object_number=116, tag_tid=29, entry_tid=12072]]], state=connected] 2024-09-14T07:06:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:28.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:28.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:28.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:28.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=12075], [object_number=118, tag_tid=29, entry_tid=12074], [object_number=117, tag_tid=29, entry_tid=12073], [object_number=116, tag_tid=29, entry_tid=12072]]], state=connected] 2024-09-14T07:06:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=12075' '!=' 'object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=12075' = 'object_number=118, tag_tid=29, entry_tid=11942' 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:28.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:28.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:28.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:28.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:28.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:28.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:28.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:28.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:28.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:28.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:28.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:28.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:28.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12075 2024-09-14T07:06:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12075 2024-09-14T07:06:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:28.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:28.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:28.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:28.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:28.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:28.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:28.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:29.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:29.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:29.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:29.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:29.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:29.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:29.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:29.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:29.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:29.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:29.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:29.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:29.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:29.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:29.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:29.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:29.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:29.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:29.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:29.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:29.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:29.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:29.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:29.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:29.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:29.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:29.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:29.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=12212], [object_number=119, tag_tid=29, entry_tid=12211], [object_number=118, tag_tid=29, entry_tid=12210], [object_number=117, tag_tid=29, entry_tid=12209]]], state=connected] 2024-09-14T07:06:29.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:29.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:29.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:29.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:29.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:29.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:29.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:29.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:29.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:29.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:29.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:29.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=12212], [object_number=119, tag_tid=29, entry_tid=12211], [object_number=118, tag_tid=29, entry_tid=12210], [object_number=117, tag_tid=29, entry_tid=12209]]], state=connected] 2024-09-14T07:06:29.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=12212' '!=' 'object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=12212' = 'object_number=119, tag_tid=29, entry_tid=12075' 2024-09-14T07:06:29.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:29.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:29.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:29.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:29.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:29.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:29.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:29.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:29.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:29.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:29.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:29.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:29.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12212 2024-09-14T07:06:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12212 2024-09-14T07:06:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:29.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:29.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:29.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:29.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:29.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:29.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:29.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:29.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:29.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:29.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:29.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:29.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:29.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:29.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:29.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:29.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:29.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:29.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:29.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:29.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:30.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:30.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:30.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:30.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:30.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:30.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:30.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:30.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:30.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:30.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:30.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:30.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:30.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=12361], [object_number=116, tag_tid=29, entry_tid=12360], [object_number=119, tag_tid=29, entry_tid=12359], [object_number=118, tag_tid=29, entry_tid=12358]]], state=connected] 2024-09-14T07:06:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:30.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:30.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=12361], [object_number=116, tag_tid=29, entry_tid=12360], [object_number=119, tag_tid=29, entry_tid=12359], [object_number=118, tag_tid=29, entry_tid=12358]]], state=connected] 2024-09-14T07:06:30.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=12361' '!=' 'object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=12361' = 'object_number=116, tag_tid=29, entry_tid=12212' 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:30.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:30.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:30.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12361 2024-09-14T07:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12361 2024-09-14T07:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:30.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:30.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:30.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:30.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:30.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:30.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:30.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:30.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:30.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:30.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:30.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:30.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:30.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:30.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:30.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:30.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:30.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:30.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:30.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:30.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:30.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=12524], [object_number=119, tag_tid=29, entry_tid=12523], [object_number=118, tag_tid=29, entry_tid=12522], [object_number=117, tag_tid=29, entry_tid=12521]]], state=connected] 2024-09-14T07:06:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:31.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:31.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:31.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:31.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:31.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:31.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:31.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:31.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:31.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:31.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=12524], [object_number=119, tag_tid=29, entry_tid=12523], [object_number=118, tag_tid=29, entry_tid=12522], [object_number=117, tag_tid=29, entry_tid=12521]]], state=connected] 2024-09-14T07:06:31.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=12524' '!=' 'object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=12524' = 'object_number=117, tag_tid=29, entry_tid=12361' 2024-09-14T07:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:31.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:31.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:31.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:31.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:31.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:31.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:31.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:31.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:31.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12524 2024-09-14T07:06:31.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12524 2024-09-14T07:06:31.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:31.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:31.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:31.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:31.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:31.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:31.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:31.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:31.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:31.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:31.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:31.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:31.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:31.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:31.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:31.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:31.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:31.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:31.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:31.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:31.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:31.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:31.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:31.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:31.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:31.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:31.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:31.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:31.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:31.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:31.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:31.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:31.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:31.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:31.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:31.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=12677], [object_number=116, tag_tid=29, entry_tid=12676], [object_number=119, tag_tid=29, entry_tid=12675], [object_number=118, tag_tid=29, entry_tid=12674]]], state=connected] 2024-09-14T07:06:31.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:31.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:31.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:31.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=12677], [object_number=116, tag_tid=29, entry_tid=12676], [object_number=119, tag_tid=29, entry_tid=12675], [object_number=118, tag_tid=29, entry_tid=12674]]], state=connected] 2024-09-14T07:06:31.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:31.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:31.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:31.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=12677' '!=' 'object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=12677' = 'object_number=116, tag_tid=29, entry_tid=12524' 2024-09-14T07:06:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:31.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:31.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:31.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12677 2024-09-14T07:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12677 2024-09-14T07:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:32.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:32.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:32.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:32.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:32.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:32.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:32.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:32.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:32.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:32.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:32.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:32.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:32.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:32.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:32.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:32.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:32.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:32.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:32.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:32.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:32.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:32.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:32.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:32.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:32.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:32.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:32.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:32.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:32.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:32.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:32.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:32.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:32.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:32.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:32.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=12839], [object_number=118, tag_tid=29, entry_tid=12838], [object_number=117, tag_tid=29, entry_tid=12837], [object_number=116, tag_tid=29, entry_tid=12836]]], state=connected] 2024-09-14T07:06:32.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:32.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:32.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:32.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:32.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:32.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:32.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:32.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:32.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:32.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:32.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:32.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:32.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:32.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=119, tag_tid=29, entry_tid=12839], [object_number=118, tag_tid=29, entry_tid=12838], [object_number=117, tag_tid=29, entry_tid=12837], [object_number=116, tag_tid=29, entry_tid=12836]]], state=connected] 2024-09-14T07:06:32.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=12839' '!=' 'object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=119, tag_tid=29, entry_tid=12839' = 'object_number=117, tag_tid=29, entry_tid=12677' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:32.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:32.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:32.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:32.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:32.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:32.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:32.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:32.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:32.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12839 2024-09-14T07:06:32.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12839 2024-09-14T07:06:32.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:32.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:32.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:32.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:32.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:32.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:32.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:32.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:32.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:32.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:32.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:32.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:32.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:32.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:32.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:32.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:32.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:32.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:32.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:32.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:32.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:32.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:32.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:32.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:32.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:32.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:32.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:33.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:33.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:33.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:33.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:33.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:33.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=12998], [object_number=117, tag_tid=29, entry_tid=12997], [object_number=116, tag_tid=29, entry_tid=12996], [object_number=119, tag_tid=29, entry_tid=12995]]], state=connected] 2024-09-14T07:06:33.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:33.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:33.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:33.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:33.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:33.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:33.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:33.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=12998], [object_number=117, tag_tid=29, entry_tid=12997], [object_number=116, tag_tid=29, entry_tid=12996], [object_number=119, tag_tid=29, entry_tid=12995]]], state=connected] 2024-09-14T07:06:33.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:33.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=12998' '!=' 'object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=12998' = 'object_number=119, tag_tid=29, entry_tid=12839' 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:33.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:33.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:33.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:33.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:33.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:33.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:33.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:33.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12998 2024-09-14T07:06:33.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 12998 2024-09-14T07:06:33.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:33.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:33.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:33.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:33.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:33.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:33.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:33.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:33.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:33.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:33.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:33.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:33.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:33.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:33.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:33.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:33.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:33.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:33.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:33.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:33.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:33.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:33.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:33.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:33.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:33.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:33.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:33.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:34.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:34.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:34.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:34.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=13134], [object_number=117, tag_tid=29, entry_tid=13133], [object_number=116, tag_tid=29, entry_tid=13132], [object_number=119, tag_tid=29, entry_tid=13131]]], state=connected] 2024-09-14T07:06:34.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=13134], [object_number=117, tag_tid=29, entry_tid=13133], [object_number=116, tag_tid=29, entry_tid=13132], [object_number=119, tag_tid=29, entry_tid=13131]]], state=connected] 2024-09-14T07:06:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=13134' '!=' 'object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=13134' = 'object_number=118, tag_tid=29, entry_tid=12998' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:34.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:34.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:34.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:34.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:34.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:34.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:34.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:34.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13134 2024-09-14T07:06:34.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13134 2024-09-14T07:06:34.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:34.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:34.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:34.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:34.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:34.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:34.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:34.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:34.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:34.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:34.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:34.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:34.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:34.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:34.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:34.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:34.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:34.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:34.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:34.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:34.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:34.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:34.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:34.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:34.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:34.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:34.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:34.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:34.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:34.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:34.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:34.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:34.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:34.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:34.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:34.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:34.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:34.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:34.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:34.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:34.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:34.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:34.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:34.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:34.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:34.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=13269], [object_number=116, tag_tid=29, entry_tid=13268], [object_number=119, tag_tid=29, entry_tid=13267], [object_number=118, tag_tid=29, entry_tid=13266]]], state=connected] 2024-09-14T07:06:34.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:34.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:34.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:34.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:34.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=13269], [object_number=116, tag_tid=29, entry_tid=13268], [object_number=119, tag_tid=29, entry_tid=13267], [object_number=118, tag_tid=29, entry_tid=13266]]], state=connected] 2024-09-14T07:06:34.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=13269' '!=' 'object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=13269' = 'object_number=118, tag_tid=29, entry_tid=13134' 2024-09-14T07:06:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:34.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:34.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:34.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:34.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:34.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:34.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13269 2024-09-14T07:06:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13269 2024-09-14T07:06:34.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:34.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:35.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:35.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:35.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:35.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:35.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:35.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:35.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:35.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:35.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:35.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:35.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:35.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:35.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:35.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:35.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:35.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:35.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:35.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:35.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:35.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:35.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:35.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:35.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:35.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:35.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:35.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:35.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:35.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:35.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:35.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:35.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=13424], [object_number=119, tag_tid=29, entry_tid=13423], [object_number=118, tag_tid=29, entry_tid=13422], [object_number=117, tag_tid=29, entry_tid=13421]]], state=connected] 2024-09-14T07:06:35.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:35.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:35.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:35.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:35.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:35.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:35.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:35.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:35.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:35.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:35.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=116, tag_tid=29, entry_tid=13424], [object_number=119, tag_tid=29, entry_tid=13423], [object_number=118, tag_tid=29, entry_tid=13422], [object_number=117, tag_tid=29, entry_tid=13421]]], state=connected] 2024-09-14T07:06:35.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=13424' '!=' 'object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=116, tag_tid=29, entry_tid=13424' = 'object_number=117, tag_tid=29, entry_tid=13269' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:35.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:35.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:35.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:35.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:35.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:35.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:35.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13424 2024-09-14T07:06:35.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13424 2024-09-14T07:06:35.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:35.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:35.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:35.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:35.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:35.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:35.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:35.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:35.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:35.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:35.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:35.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:35.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:35.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:35.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:35.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:35.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:36.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:36.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:36.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:36.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:36.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:36.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:36.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:36.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:36.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:36.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=13566], [object_number=117, tag_tid=29, entry_tid=13565], [object_number=116, tag_tid=29, entry_tid=13564], [object_number=119, tag_tid=29, entry_tid=13563]]], state=connected] 2024-09-14T07:06:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:36.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:36.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:36.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:36.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:36.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:36.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:36.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:36.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:36.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=13566], [object_number=117, tag_tid=29, entry_tid=13565], [object_number=116, tag_tid=29, entry_tid=13564], [object_number=119, tag_tid=29, entry_tid=13563]]], state=connected] 2024-09-14T07:06:36.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:36.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=13566' '!=' 'object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=13566' = 'object_number=116, tag_tid=29, entry_tid=13424' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:36.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:36.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:36.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:36.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:36.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:36.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:36.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:36.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:36.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:36.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:36.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13566 2024-09-14T07:06:36.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13566 2024-09-14T07:06:36.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:36.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:36.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:36.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:36.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:36.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:36.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:36.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:36.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:36.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:36.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:36.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:36.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:36.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:36.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:36.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:36.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:36.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:36.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:36.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:36.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:37.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:37.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:37.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:37.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:37.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:37.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:37.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:37.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:37.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:37.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:37.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:37.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:37.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:37.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:37.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:37.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:37.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:37.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:37.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:37.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:37.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=13734], [object_number=117, tag_tid=29, entry_tid=13733], [object_number=116, tag_tid=29, entry_tid=13732], [object_number=119, tag_tid=29, entry_tid=13731]]], state=connected] 2024-09-14T07:06:37.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:37.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:37.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:37.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:37.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=118, tag_tid=29, entry_tid=13734], [object_number=117, tag_tid=29, entry_tid=13733], [object_number=116, tag_tid=29, entry_tid=13732], [object_number=119, tag_tid=29, entry_tid=13731]]], state=connected] 2024-09-14T07:06:37.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:37.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:37.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=13734' '!=' 'object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:37.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:37.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=118, tag_tid=29, entry_tid=13734' = 'object_number=118, tag_tid=29, entry_tid=13566' 2024-09-14T07:06:37.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:37.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:37.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:37.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:37.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:37.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:37.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:37.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:37.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:37.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:37.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:37.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:37.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:37.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13734 2024-09-14T07:06:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13734 2024-09-14T07:06:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:37.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:37.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:37.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:37.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:37.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:37.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:37.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:37.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:37.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:37.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:37.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:37.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:37.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:37.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:37.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:37.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:37.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:37.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:37.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:37.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:37.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:37.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:37.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:37.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:37.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:37.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:37.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:37.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:37.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:37.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:38.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=13889], [object_number=116, tag_tid=29, entry_tid=13888], [object_number=119, tag_tid=29, entry_tid=13887], [object_number=118, tag_tid=29, entry_tid=13886]]], state=connected] 2024-09-14T07:06:38.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:38.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:38.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:38.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:38.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:38.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 29 2024-09-14T07:06:38.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:38.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:38.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:38.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=117, tag_tid=29, entry_tid=13889], [object_number=116, tag_tid=29, entry_tid=13888], [object_number=119, tag_tid=29, entry_tid=13887], [object_number=118, tag_tid=29, entry_tid=13886]]], state=connected] 2024-09-14T07:06:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=13889' '!=' 'object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=117, tag_tid=29, entry_tid=13889' = 'object_number=118, tag_tid=29, entry_tid=13734' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:38.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:38.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:38.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:38.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13889 2024-09-14T07:06:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13889 2024-09-14T07:06:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:38.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:38.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:38.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:38.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:38.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:38.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:38.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:38.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:38.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:38.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:38.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:38.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:38.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:38.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:38.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:38.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:38.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:38.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:38.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:38.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:38.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:38.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:38.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:38.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:38.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:38.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:38.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:38.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:38.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:38.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:38.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=13994], [object_number=121, tag_tid=29, entry_tid=13993], [object_number=120, tag_tid=29, entry_tid=13992], [object_number=123, tag_tid=29, entry_tid=13991]]], state=connected] 2024-09-14T07:06:38.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:38.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:38.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:38.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:38.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:38.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=13994], [object_number=121, tag_tid=29, entry_tid=13993], [object_number=120, tag_tid=29, entry_tid=13992], [object_number=123, tag_tid=29, entry_tid=13991]]], state=connected] 2024-09-14T07:06:38.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:38.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=13994' '!=' 'object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=13994' = 'object_number=117, tag_tid=29, entry_tid=13889' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:38.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:38.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:38.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:38.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:38.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:38.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:38.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:38.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:38.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=13994 2024-09-14T07:06:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 13994 2024-09-14T07:06:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:39.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:39.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:39.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:39.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:39.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:39.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:39.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:39.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:39.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:39.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:39.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:39.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:39.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:39.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:39.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:39.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:39.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:39.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:39.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:39.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:39.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:39.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:39.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:39.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:39.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:39.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:39.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:39.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:39.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:39.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:39.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:39.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:39.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:39.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:39.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=14137], [object_number=120, tag_tid=29, entry_tid=14136], [object_number=123, tag_tid=29, entry_tid=14135], [object_number=122, tag_tid=29, entry_tid=14134]]], state=connected] 2024-09-14T07:06:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:39.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:39.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:39.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:39.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=14137], [object_number=120, tag_tid=29, entry_tid=14136], [object_number=123, tag_tid=29, entry_tid=14135], [object_number=122, tag_tid=29, entry_tid=14134]]], state=connected] 2024-09-14T07:06:39.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:39.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:39.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=14137' '!=' 'object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=14137' = 'object_number=122, tag_tid=29, entry_tid=13994' 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:39.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:39.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:39.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:39.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:39.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:39.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:39.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:39.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:39.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14137 2024-09-14T07:06:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14137 2024-09-14T07:06:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:39.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:39.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:39.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:39.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:39.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:39.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:39.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:39.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:39.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:39.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:39.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:39.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:40.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:40.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:40.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:40.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:40.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:40.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:40.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:40.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:40.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:40.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:40.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:40.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:40.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:40.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=14277], [object_number=120, tag_tid=29, entry_tid=14276], [object_number=123, tag_tid=29, entry_tid=14275], [object_number=122, tag_tid=29, entry_tid=14274]]], state=connected] 2024-09-14T07:06:40.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:40.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:40.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:40.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:40.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:40.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:40.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:40.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=14277], [object_number=120, tag_tid=29, entry_tid=14276], [object_number=123, tag_tid=29, entry_tid=14275], [object_number=122, tag_tid=29, entry_tid=14274]]], state=connected] 2024-09-14T07:06:40.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:40.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:40.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:40.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=14277' '!=' 'object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:40.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:40.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=14277' = 'object_number=121, tag_tid=29, entry_tid=14137' 2024-09-14T07:06:40.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:40.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:40.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:40.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:40.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:40.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:40.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:40.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:40.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14277 2024-09-14T07:06:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14277 2024-09-14T07:06:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:40.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:40.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:40.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:40.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:40.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:40.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:40.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:40.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:40.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:40.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:40.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:40.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:40.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:40.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:40.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:40.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:40.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:40.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:40.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:40.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:40.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:40.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:40.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:40.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:40.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:41.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:41.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:41.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:41.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:41.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=14411], [object_number=122, tag_tid=29, entry_tid=14410], [object_number=121, tag_tid=29, entry_tid=14409], [object_number=120, tag_tid=29, entry_tid=14408]]], state=connected] 2024-09-14T07:06:41.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=14411], [object_number=122, tag_tid=29, entry_tid=14410], [object_number=121, tag_tid=29, entry_tid=14409], [object_number=120, tag_tid=29, entry_tid=14408]]], state=connected] 2024-09-14T07:06:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=14411' '!=' 'object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=14411' = 'object_number=121, tag_tid=29, entry_tid=14277' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:41.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:41.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:41.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:41.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:41.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:41.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:41.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14411 2024-09-14T07:06:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14411 2024-09-14T07:06:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:41.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:41.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:41.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:41.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:41.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:41.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:41.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:41.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:41.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:41.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:41.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:41.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:41.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:41.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:41.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:41.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:41.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:41.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:41.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:41.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:41.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:41.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:41.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=14550], [object_number=121, tag_tid=29, entry_tid=14549], [object_number=120, tag_tid=29, entry_tid=14548], [object_number=123, tag_tid=29, entry_tid=14547]]], state=connected] 2024-09-14T07:06:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:41.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:41.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:41.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:41.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:41.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=14550], [object_number=121, tag_tid=29, entry_tid=14549], [object_number=120, tag_tid=29, entry_tid=14548], [object_number=123, tag_tid=29, entry_tid=14547]]], state=connected] 2024-09-14T07:06:41.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=14550' '!=' 'object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=14550' = 'object_number=123, tag_tid=29, entry_tid=14411' 2024-09-14T07:06:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:41.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:41.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:41.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:41.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:41.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:41.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:41.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:41.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:41.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14550 2024-09-14T07:06:41.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14550 2024-09-14T07:06:41.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:41.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:41.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:42.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:42.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:42.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:42.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:42.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:42.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:42.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:42.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:42.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:42.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:42.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:42.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:42.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:42.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:42.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:42.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:42.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:42.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:42.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:42.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:42.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:42.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:42.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:42.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:42.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:42.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:42.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:42.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:42.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:42.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:42.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:42.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:42.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:42.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:42.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:42.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:42.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:42.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:42.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:42.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:42.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:42.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:42.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=14692], [object_number=123, tag_tid=29, entry_tid=14691], [object_number=122, tag_tid=29, entry_tid=14690], [object_number=121, tag_tid=29, entry_tid=14689]]], state=connected] 2024-09-14T07:06:42.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:42.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:42.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:42.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:42.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:42.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:42.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:42.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:42.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:42.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:42.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=14692], [object_number=123, tag_tid=29, entry_tid=14691], [object_number=122, tag_tid=29, entry_tid=14690], [object_number=121, tag_tid=29, entry_tid=14689]]], state=connected] 2024-09-14T07:06:42.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=14692' '!=' 'object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=14692' = 'object_number=122, tag_tid=29, entry_tid=14550' 2024-09-14T07:06:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:42.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:42.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:42.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:42.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:42.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14692 2024-09-14T07:06:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14692 2024-09-14T07:06:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:42.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:43.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:43.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:43.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:43.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:43.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:43.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:43.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:43.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:43.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:43.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:43.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:43.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:43.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:43.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=14830], [object_number=121, tag_tid=29, entry_tid=14829], [object_number=120, tag_tid=29, entry_tid=14828], [object_number=123, tag_tid=29, entry_tid=14827]]], state=connected] 2024-09-14T07:06:43.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:43.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:43.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:43.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:43.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:43.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:43.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=14830], [object_number=121, tag_tid=29, entry_tid=14829], [object_number=120, tag_tid=29, entry_tid=14828], [object_number=123, tag_tid=29, entry_tid=14827]]], state=connected] 2024-09-14T07:06:43.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:43.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=14830' '!=' 'object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=14830' = 'object_number=120, tag_tid=29, entry_tid=14692' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:43.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:43.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:43.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:43.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:43.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:43.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:43.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:43.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:43.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14830 2024-09-14T07:06:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14830 2024-09-14T07:06:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:43.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:43.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:43.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:43.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:43.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:43.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:43.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:43.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:43.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:43.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:43.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:43.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:43.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:44.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:44.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:44.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:44.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:44.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:44.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:44.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=14984], [object_number=123, tag_tid=29, entry_tid=14983], [object_number=122, tag_tid=29, entry_tid=14982], [object_number=121, tag_tid=29, entry_tid=14981]]], state=connected] 2024-09-14T07:06:44.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:44.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:44.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=14984], [object_number=123, tag_tid=29, entry_tid=14983], [object_number=122, tag_tid=29, entry_tid=14982], [object_number=121, tag_tid=29, entry_tid=14981]]], state=connected] 2024-09-14T07:06:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=14984' '!=' 'object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=14984' = 'object_number=122, tag_tid=29, entry_tid=14830' 2024-09-14T07:06:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:44.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:44.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:44.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:44.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14984 2024-09-14T07:06:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 14984 2024-09-14T07:06:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:44.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:44.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:44.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:44.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:44.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:44.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:44.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:44.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:44.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:44.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:44.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:44.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:44.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:44.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:44.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:44.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15126], [object_number=121, tag_tid=29, entry_tid=15125], [object_number=120, tag_tid=29, entry_tid=15124], [object_number=123, tag_tid=29, entry_tid=15123]]], state=connected] 2024-09-14T07:06:44.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15126], [object_number=121, tag_tid=29, entry_tid=15125], [object_number=120, tag_tid=29, entry_tid=15124], [object_number=123, tag_tid=29, entry_tid=15123]]], state=connected] 2024-09-14T07:06:44.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15126' '!=' 'object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15126' = 'object_number=120, tag_tid=29, entry_tid=14984' 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:44.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15126 2024-09-14T07:06:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15126 2024-09-14T07:06:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:45.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:45.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:45.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:45.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:45.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:45.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:45.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:45.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:45.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:45.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:45.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:45.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:45.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:45.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:45.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:45.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:45.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:45.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:45.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:45.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:45.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:45.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:45.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:45.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:45.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:45.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:45.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:45.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:45.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:45.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:45.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:45.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:45.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:45.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:45.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:45.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:45.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:45.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:45.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15266], [object_number=121, tag_tid=29, entry_tid=15265], [object_number=120, tag_tid=29, entry_tid=15264], [object_number=123, tag_tid=29, entry_tid=15263]]], state=connected] 2024-09-14T07:06:45.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:45.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:45.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:45.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:45.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:45.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:45.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:45.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:45.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:45.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:45.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15266], [object_number=121, tag_tid=29, entry_tid=15265], [object_number=120, tag_tid=29, entry_tid=15264], [object_number=123, tag_tid=29, entry_tid=15263]]], state=connected] 2024-09-14T07:06:45.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15266' '!=' 'object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15266' = 'object_number=122, tag_tid=29, entry_tid=15126' 2024-09-14T07:06:45.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:45.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:45.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:45.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:45.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:45.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:45.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:45.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:45.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:45.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:45.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:45.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:45.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15266 2024-09-14T07:06:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15266 2024-09-14T07:06:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:45.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:45.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:45.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:45.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:45.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:45.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:45.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:45.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:45.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:46.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:46.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:46.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:46.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:46.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:46.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:46.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:46.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:46.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:46.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:46.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:46.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:46.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:46.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:46.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:46.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:46.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:46.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:46.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:46.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:46.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:46.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:46.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=15413], [object_number=120, tag_tid=29, entry_tid=15412], [object_number=123, tag_tid=29, entry_tid=15411], [object_number=122, tag_tid=29, entry_tid=15410]]], state=connected] 2024-09-14T07:06:46.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:46.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:46.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:46.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:46.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:46.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=15413], [object_number=120, tag_tid=29, entry_tid=15412], [object_number=123, tag_tid=29, entry_tid=15411], [object_number=122, tag_tid=29, entry_tid=15410]]], state=connected] 2024-09-14T07:06:46.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=15413' '!=' 'object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=15413' = 'object_number=122, tag_tid=29, entry_tid=15266' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:46.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:46.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:46.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:46.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:46.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:46.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:46.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:46.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:46.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:46.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:46.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:46.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:46.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15413 2024-09-14T07:06:46.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15413 2024-09-14T07:06:46.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:46.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:46.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:46.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:46.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:46.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:46.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:46.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:46.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:46.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:46.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:46.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:46.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:46.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:46.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:46.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:46.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:46.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:46.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:46.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:46.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:46.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:46.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:46.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:46.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:46.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:46.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:46.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:46.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:47.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:47.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:47.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:47.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:47.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:47.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:47.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:47.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:47.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15558], [object_number=121, tag_tid=29, entry_tid=15557], [object_number=120, tag_tid=29, entry_tid=15556], [object_number=123, tag_tid=29, entry_tid=15555]]], state=connected] 2024-09-14T07:06:47.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:47.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:47.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:47.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:47.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:47.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:47.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15558], [object_number=121, tag_tid=29, entry_tid=15557], [object_number=120, tag_tid=29, entry_tid=15556], [object_number=123, tag_tid=29, entry_tid=15555]]], state=connected] 2024-09-14T07:06:47.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15558' '!=' 'object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15558' = 'object_number=121, tag_tid=29, entry_tid=15413' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:47.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:47.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:47.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:47.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:47.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:47.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:47.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:47.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:47.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:47.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15558 2024-09-14T07:06:47.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15558 2024-09-14T07:06:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:47.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:47.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:47.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:47.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:47.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:47.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:47.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:47.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:47.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:47.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:47.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:47.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:47.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:47.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:47.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:47.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:47.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:47.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:47.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:47.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:47.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:47.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:47.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:47.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:47.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=15700], [object_number=123, tag_tid=29, entry_tid=15699], [object_number=122, tag_tid=29, entry_tid=15698], [object_number=121, tag_tid=29, entry_tid=15697]]], state=connected] 2024-09-14T07:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:47.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:47.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:47.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:48.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:48.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:48.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:48.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=15700], [object_number=123, tag_tid=29, entry_tid=15699], [object_number=122, tag_tid=29, entry_tid=15698], [object_number=121, tag_tid=29, entry_tid=15697]]], state=connected] 2024-09-14T07:06:48.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=15700' '!=' 'object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=15700' = 'object_number=122, tag_tid=29, entry_tid=15558' 2024-09-14T07:06:48.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:48.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:48.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:48.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:48.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15700 2024-09-14T07:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15700 2024-09-14T07:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:48.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:48.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:48.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:48.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:48.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:48.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:48.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:48.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:48.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:48.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:48.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:48.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:48.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:48.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:48.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:48.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:48.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:48.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:48.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:48.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:48.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:48.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:48.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:48.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:48.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:48.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:48.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:48.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:48.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15830], [object_number=121, tag_tid=29, entry_tid=15829], [object_number=120, tag_tid=29, entry_tid=15828], [object_number=123, tag_tid=29, entry_tid=15827]]], state=connected] 2024-09-14T07:06:48.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:48.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:48.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:48.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:48.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:48.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:48.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=15830], [object_number=121, tag_tid=29, entry_tid=15829], [object_number=120, tag_tid=29, entry_tid=15828], [object_number=123, tag_tid=29, entry_tid=15827]]], state=connected] 2024-09-14T07:06:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15830' '!=' 'object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=15830' = 'object_number=120, tag_tid=29, entry_tid=15700' 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:48.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15830 2024-09-14T07:06:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15830 2024-09-14T07:06:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:48.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:48.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:48.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:49.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:49.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:49.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:49.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:49.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:49.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:49.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:49.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:49.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:49.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:49.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:49.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:49.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:49.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:49.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:49.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:49.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:49.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:49.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:49.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:49.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:49.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:49.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:49.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:49.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:49.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:49.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:49.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:49.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:49.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:49.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=15961], [object_number=120, tag_tid=29, entry_tid=15960], [object_number=123, tag_tid=29, entry_tid=15959], [object_number=122, tag_tid=29, entry_tid=15958]]], state=connected] 2024-09-14T07:06:49.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:49.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:49.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:49.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=15961], [object_number=120, tag_tid=29, entry_tid=15960], [object_number=123, tag_tid=29, entry_tid=15959], [object_number=122, tag_tid=29, entry_tid=15958]]], state=connected] 2024-09-14T07:06:49.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=15961' '!=' 'object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=15961' = 'object_number=122, tag_tid=29, entry_tid=15830' 2024-09-14T07:06:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=15961 2024-09-14T07:06:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 15961 2024-09-14T07:06:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:49.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:49.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:49.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:49.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:49.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:49.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:49.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:49.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:49.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:49.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:49.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:49.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:49.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:49.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:49.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:49.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:49.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:49.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:49.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:49.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:49.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:49.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:49.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:49.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:49.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:49.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:50.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:50.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:50.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:50.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:50.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:50.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:50.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:50.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:50.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:50.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:50.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:50.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:50.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:50.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:50.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=16095], [object_number=122, tag_tid=29, entry_tid=16094], [object_number=121, tag_tid=29, entry_tid=16093], [object_number=120, tag_tid=29, entry_tid=16092]]], state=connected] 2024-09-14T07:06:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:50.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=16095], [object_number=122, tag_tid=29, entry_tid=16094], [object_number=121, tag_tid=29, entry_tid=16093], [object_number=120, tag_tid=29, entry_tid=16092]]], state=connected] 2024-09-14T07:06:50.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=16095' '!=' 'object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=16095' = 'object_number=121, tag_tid=29, entry_tid=15961' 2024-09-14T07:06:50.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:50.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:50.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:50.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:50.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:50.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:50.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:50.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:50.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:50.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:50.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:50.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:50.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:50.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16095 2024-09-14T07:06:50.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16095 2024-09-14T07:06:50.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:50.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:50.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:50.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:50.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:50.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:50.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:50.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:50.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:50.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:50.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:50.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:50.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:50.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:50.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:50.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:50.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:50.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:50.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:50.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:50.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:50.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:50.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:50.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:50.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:50.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:50.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:50.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:50.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=16234], [object_number=121, tag_tid=29, entry_tid=16233], [object_number=120, tag_tid=29, entry_tid=16232], [object_number=123, tag_tid=29, entry_tid=16231]]], state=connected] 2024-09-14T07:06:50.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:50.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:50.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:50.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:50.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=16234], [object_number=121, tag_tid=29, entry_tid=16233], [object_number=120, tag_tid=29, entry_tid=16232], [object_number=123, tag_tid=29, entry_tid=16231]]], state=connected] 2024-09-14T07:06:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=16234' '!=' 'object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=16234' = 'object_number=123, tag_tid=29, entry_tid=16095' 2024-09-14T07:06:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:51.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:51.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:51.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:51.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:51.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16234 2024-09-14T07:06:51.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16234 2024-09-14T07:06:51.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:51.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:51.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:51.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:51.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:51.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:51.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:51.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:51.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:51.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:51.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:51.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:51.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:51.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:51.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:51.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:51.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:51.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:51.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:51.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:51.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:51.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:51.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:51.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:51.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=16384], [object_number=123, tag_tid=29, entry_tid=16383], [object_number=122, tag_tid=29, entry_tid=16382], [object_number=121, tag_tid=29, entry_tid=16381]]], state=connected] 2024-09-14T07:06:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:51.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:51.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:51.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:51.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=16384], [object_number=123, tag_tid=29, entry_tid=16383], [object_number=122, tag_tid=29, entry_tid=16382], [object_number=121, tag_tid=29, entry_tid=16381]]], state=connected] 2024-09-14T07:06:51.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=16384' '!=' 'object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=16384' = 'object_number=122, tag_tid=29, entry_tid=16234' 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:51.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:51.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:51.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:51.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:51.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:51.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16384 2024-09-14T07:06:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16384 2024-09-14T07:06:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:51.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:52.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:52.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:52.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:52.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:52.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:52.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:52.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:52.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:52.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:52.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:52.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:52.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:52.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:52.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:52.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:52.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:52.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=16545], [object_number=120, tag_tid=29, entry_tid=16544], [object_number=123, tag_tid=29, entry_tid=16543], [object_number=122, tag_tid=29, entry_tid=16542]]], state=connected] 2024-09-14T07:06:52.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=16545], [object_number=120, tag_tid=29, entry_tid=16544], [object_number=123, tag_tid=29, entry_tid=16543], [object_number=122, tag_tid=29, entry_tid=16542]]], state=connected] 2024-09-14T07:06:52.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:52.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:52.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=16545' '!=' 'object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=16545' = 'object_number=120, tag_tid=29, entry_tid=16384' 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:52.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:52.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:52.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:52.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:52.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:52.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:52.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:52.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:52.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:52.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:52.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:52.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16545 2024-09-14T07:06:52.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16545 2024-09-14T07:06:52.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:52.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:52.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:52.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:52.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:52.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:52.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:52.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:52.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:52.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:52.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:52.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:52.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:52.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:52.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:52.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:53.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:53.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:53.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:53.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:53.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:53.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:53.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:53.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:53.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:53.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:53.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:53.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:53.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=16707], [object_number=122, tag_tid=29, entry_tid=16706], [object_number=121, tag_tid=29, entry_tid=16705], [object_number=120, tag_tid=29, entry_tid=16704]]], state=connected] 2024-09-14T07:06:53.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:53.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:53.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:53.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:53.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:53.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:53.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=16707], [object_number=122, tag_tid=29, entry_tid=16706], [object_number=121, tag_tid=29, entry_tid=16705], [object_number=120, tag_tid=29, entry_tid=16704]]], state=connected] 2024-09-14T07:06:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=16707' '!=' 'object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=16707' = 'object_number=121, tag_tid=29, entry_tid=16545' 2024-09-14T07:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:53.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:53.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:53.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:53.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:53.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16707 2024-09-14T07:06:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16707 2024-09-14T07:06:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:53.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:53.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:53.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:53.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:53.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:53.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:53.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:53.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:53.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:53.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:53.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:53.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:53.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:53.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:53.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:53.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:53.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:53.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:53.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:53.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:53.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:53.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:53.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:53.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:53.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:53.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:53.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:53.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:53.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:53.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:53.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:53.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:53.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:53.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=16835], [object_number=122, tag_tid=29, entry_tid=16834], [object_number=121, tag_tid=29, entry_tid=16833], [object_number=120, tag_tid=29, entry_tid=16832]]], state=connected] 2024-09-14T07:06:54.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:54.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:54.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=16835], [object_number=122, tag_tid=29, entry_tid=16834], [object_number=121, tag_tid=29, entry_tid=16833], [object_number=120, tag_tid=29, entry_tid=16832]]], state=connected] 2024-09-14T07:06:54.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=16835' '!=' 'object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=16835' = 'object_number=123, tag_tid=29, entry_tid=16707' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:54.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:54.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:54.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:54.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:54.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:54.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:54.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16835 2024-09-14T07:06:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16835 2024-09-14T07:06:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:54.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:54.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:54.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:54.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:54.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:54.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:54.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:54.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:54.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:54.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:54.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:54.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:54.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:54.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:54.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:54.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:54.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:54.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:54.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:54.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:54.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:54.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:54.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:54.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:54.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:54.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:54.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:54.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:54.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:54.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:54.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:54.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:54.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:54.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=16992], [object_number=123, tag_tid=29, entry_tid=16991], [object_number=122, tag_tid=29, entry_tid=16990], [object_number=121, tag_tid=29, entry_tid=16989]]], state=connected] 2024-09-14T07:06:54.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:54.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:54.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:54.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:54.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:54.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:54.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:54.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=16992], [object_number=123, tag_tid=29, entry_tid=16991], [object_number=122, tag_tid=29, entry_tid=16990], [object_number=121, tag_tid=29, entry_tid=16989]]], state=connected] 2024-09-14T07:06:54.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=16992' '!=' 'object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=16992' = 'object_number=123, tag_tid=29, entry_tid=16835' 2024-09-14T07:06:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:54.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:54.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:54.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:54.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:54.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:54.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:54.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:54.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:54.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:54.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:54.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=16992 2024-09-14T07:06:54.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 16992 2024-09-14T07:06:54.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:54.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:54.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:55.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:55.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:55.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:55.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:55.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:55.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:55.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:55.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:55.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:55.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:55.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:55.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:55.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:55.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:55.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:55.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:55.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:55.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:55.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:55.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:55.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:55.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:55.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17125], [object_number=120, tag_tid=29, entry_tid=17124], [object_number=123, tag_tid=29, entry_tid=17123], [object_number=122, tag_tid=29, entry_tid=17122]]], state=connected] 2024-09-14T07:06:55.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:55.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:55.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:55.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:55.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:55.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17125], [object_number=120, tag_tid=29, entry_tid=17124], [object_number=123, tag_tid=29, entry_tid=17123], [object_number=122, tag_tid=29, entry_tid=17122]]], state=connected] 2024-09-14T07:06:55.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17125' '!=' 'object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17125' = 'object_number=120, tag_tid=29, entry_tid=16992' 2024-09-14T07:06:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:55.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:55.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:55.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:55.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:55.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:55.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:55.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17125 2024-09-14T07:06:55.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 17125 2024-09-14T07:06:55.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:55.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:55.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:56.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:56.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:56.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:56.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:56.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:56.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:56.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:56.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:56.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:56.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:56.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:56.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:56.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:56.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:56.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:56.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:56.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:56.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:56.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:56.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:56.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:56.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:56.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:56.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:56.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:56.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:56.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:56.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:56.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:56.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17301], [object_number=120, tag_tid=29, entry_tid=17300], [object_number=123, tag_tid=29, entry_tid=17299], [object_number=122, tag_tid=29, entry_tid=17298]]], state=connected] 2024-09-14T07:06:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:56.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:56.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:56.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:56.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:56.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:56.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17301], [object_number=120, tag_tid=29, entry_tid=17300], [object_number=123, tag_tid=29, entry_tid=17299], [object_number=122, tag_tid=29, entry_tid=17298]]], state=connected] 2024-09-14T07:06:56.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17301' '!=' 'object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17301' = 'object_number=121, tag_tid=29, entry_tid=17125' 2024-09-14T07:06:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:56.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:56.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:56.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:56.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:56.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:56.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:56.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:56.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:56.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17301 2024-09-14T07:06:56.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 17301 2024-09-14T07:06:56.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:56.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:56.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:56.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:56.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:56.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:56.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:56.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:56.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:56.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:56.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:56.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:56.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:56.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:56.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:56.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:57.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:57.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:57.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:57.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:57.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:57.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:57.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:57.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:57.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:57.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17457], [object_number=120, tag_tid=29, entry_tid=17456], [object_number=123, tag_tid=29, entry_tid=17455], [object_number=122, tag_tid=29, entry_tid=17454]]], state=connected] 2024-09-14T07:06:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:57.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:57.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:57.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:57.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:57.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:57.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:57.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:57.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17457], [object_number=120, tag_tid=29, entry_tid=17456], [object_number=123, tag_tid=29, entry_tid=17455], [object_number=122, tag_tid=29, entry_tid=17454]]], state=connected] 2024-09-14T07:06:57.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:57.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:57.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17457' '!=' 'object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:57.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:57.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17457' = 'object_number=121, tag_tid=29, entry_tid=17301' 2024-09-14T07:06:57.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:57.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:57.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:57.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:57.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:57.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:57.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:57.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:57.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:57.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:57.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:57.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:57.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17457 2024-09-14T07:06:57.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 17457 2024-09-14T07:06:57.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:57.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:57.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:57.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:57.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:57.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:57.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:57.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:57.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:57.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:57.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:57.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:57.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:57.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:57.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:57.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:57.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:57.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:57.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:57.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:57.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:57.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:57.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:57.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:57.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:57.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:57.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:57.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:57.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:57.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:57.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:57.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:57.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:57.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:57.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:57.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:58.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:58.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:58.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:58.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=17598], [object_number=121, tag_tid=29, entry_tid=17597], [object_number=120, tag_tid=29, entry_tid=17596], [object_number=123, tag_tid=29, entry_tid=17595]]], state=connected] 2024-09-14T07:06:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:58.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:58.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:58.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:58.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=122, tag_tid=29, entry_tid=17598], [object_number=121, tag_tid=29, entry_tid=17597], [object_number=120, tag_tid=29, entry_tid=17596], [object_number=123, tag_tid=29, entry_tid=17595]]], state=connected] 2024-09-14T07:06:58.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=17598' '!=' 'object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=122, tag_tid=29, entry_tid=17598' = 'object_number=121, tag_tid=29, entry_tid=17457' 2024-09-14T07:06:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:58.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:58.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:58.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:58.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:58.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:58.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17598 2024-09-14T07:06:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 17598 2024-09-14T07:06:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:58.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:58.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:58.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:58.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:58.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:58.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:58.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:58.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:58.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:58.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:58.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:58.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:58.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:58.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:58.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:58.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:58.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:58.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:58.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:58.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:58.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:58.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:58.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:58.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:58.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:58.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:58.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:58.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:58.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:58.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:58.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:58.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:58.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:58.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:58.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:58.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17737], [object_number=120, tag_tid=29, entry_tid=17736], [object_number=123, tag_tid=29, entry_tid=17735], [object_number=122, tag_tid=29, entry_tid=17734]]], state=connected] 2024-09-14T07:06:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:58.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:58.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:58.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:58.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:58.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=121, tag_tid=29, entry_tid=17737], [object_number=120, tag_tid=29, entry_tid=17736], [object_number=123, tag_tid=29, entry_tid=17735], [object_number=122, tag_tid=29, entry_tid=17734]]], state=connected] 2024-09-14T07:06:58.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:58.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:58.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:58.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17737' '!=' 'object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:58.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=121, tag_tid=29, entry_tid=17737' = 'object_number=122, tag_tid=29, entry_tid=17598' 2024-09-14T07:06:58.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:58.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:58.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:58.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:58.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:58.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:58.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:58.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:58.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:58.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:58.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17737 2024-09-14T07:06:58.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 17737 2024-09-14T07:06:58.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:58.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:58.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:59.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:59.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:59.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:59.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:59.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:59.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:59.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:59.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:59.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:59.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:59.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:59.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:59.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:59.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:59.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:06:59.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:59.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:59.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:59.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:59.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:06:59.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:06:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:06:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:06:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:06:59.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:06:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:59.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:06:59.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:06:59.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:06:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:06:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:59.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:06:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:06:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:59.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:59.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:59.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=17880], [object_number=123, tag_tid=29, entry_tid=17879], [object_number=122, tag_tid=29, entry_tid=17878], [object_number=121, tag_tid=29, entry_tid=17877]]], state=connected] 2024-09-14T07:06:59.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:59.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:06:59.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:06:59.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:06:59.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:06:59.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:06:59.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:59.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:06:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:06:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:06:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:06:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:06:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=120, tag_tid=29, entry_tid=17880], [object_number=123, tag_tid=29, entry_tid=17879], [object_number=122, tag_tid=29, entry_tid=17878], [object_number=121, tag_tid=29, entry_tid=17877]]], state=connected] 2024-09-14T07:06:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:06:59.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:06:59.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:06:59.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=17880' '!=' 'object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:59.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:06:59.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=120, tag_tid=29, entry_tid=17880' = 'object_number=121, tag_tid=29, entry_tid=17737' 2024-09-14T07:06:59.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:06:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:59.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:06:59.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:06:59.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:06:59.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:59.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:06:59.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:06:59.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:59.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:06:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:06:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:06:59.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:06:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=17880 2024-09-14T07:06:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 17880 2024-09-14T07:06:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:06:59.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:06:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:06:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:06:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:06:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:06:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:06:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:06:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:06:59.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:06:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:06:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:06:59.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:59.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:59.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:06:59.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:06:59.850 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9597296528320262) > weight (0.25) 2024-09-14T07:06:59.850 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T07:06:59.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T07:06:59.851 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:06:59.851 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:06:59.855+0000 7fa5bab26640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 72863) UID: 1000 2024-09-14T07:06:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:06:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:06:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:06:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:06:59.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:06:59.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:06:59.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:06:59.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:59.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:59.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:06:59.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:00.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:00.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:00.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:00.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:00.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:00.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:07:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=18051], [object_number=122, tag_tid=29, entry_tid=18050], [object_number=121, tag_tid=29, entry_tid=18049], [object_number=120, tag_tid=29, entry_tid=18048]]], state=connected] 2024-09-14T07:07:00.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:00.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 30 2024-09-14T07:07:00.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:00.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:00.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:00.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=123, tag_tid=29, entry_tid=18051], [object_number=122, tag_tid=29, entry_tid=18050], [object_number=121, tag_tid=29, entry_tid=18049], [object_number=120, tag_tid=29, entry_tid=18048]]], state=connected] 2024-09-14T07:07:00.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:00.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:00.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=18051' '!=' 'object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:07:00.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:00.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=123, tag_tid=29, entry_tid=18051' = 'object_number=120, tag_tid=29, entry_tid=17880' 2024-09-14T07:07:00.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:00.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:00.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:00.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:00.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:00.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18051 2024-09-14T07:07:00.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18051 2024-09-14T07:07:00.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:00.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:00.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:00.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:00.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:00.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:00.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:00.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:00.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:00.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:00.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:00.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:00.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:00.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:00.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:00.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:00.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:01.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:01.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:01.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:01.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:01.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18200], [object_number=127, tag_tid=29, entry_tid=18199], [object_number=126, tag_tid=29, entry_tid=18198], [object_number=125, tag_tid=29, entry_tid=18197]]], state=connected] 2024-09-14T07:07:01.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:01.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18200], [object_number=127, tag_tid=29, entry_tid=18199], [object_number=126, tag_tid=29, entry_tid=18198], [object_number=125, tag_tid=29, entry_tid=18197]]], state=connected] 2024-09-14T07:07:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:01.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:01.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:01.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18200' '!=' 'object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18200' = 'object_number=123, tag_tid=29, entry_tid=18051' 2024-09-14T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:01.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:01.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:01.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:01.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:01.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:01.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:01.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:01.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:01.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18200 2024-09-14T07:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18200 2024-09-14T07:07:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:01.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:01.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:01.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:01.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:01.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:01.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:01.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:01.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:01.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:01.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:01.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:01.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:01.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:01.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:01.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:01.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:01.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:01.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:01.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:01.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:01.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:01.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:01.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:01.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:01.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:01.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:01.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:01.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:01.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:01.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:01.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:01.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:01.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:01.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:01.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18348], [object_number=127, tag_tid=29, entry_tid=18347], [object_number=126, tag_tid=29, entry_tid=18346], [object_number=125, tag_tid=29, entry_tid=18345]]], state=connected] 2024-09-14T07:07:02.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:02.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:02.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:02.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:02.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:02.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:02.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:02.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:02.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:02.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18348], [object_number=127, tag_tid=29, entry_tid=18347], [object_number=126, tag_tid=29, entry_tid=18346], [object_number=125, tag_tid=29, entry_tid=18345]]], state=connected] 2024-09-14T07:07:02.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18348' '!=' 'object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18348' = 'object_number=124, tag_tid=29, entry_tid=18200' 2024-09-14T07:07:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18348 2024-09-14T07:07:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18348 2024-09-14T07:07:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:02.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:02.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:02.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:02.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:02.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:02.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:02.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:02.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:02.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:02.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:02.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:02.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:02.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:02.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:02.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:02.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:02.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:02.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:02.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:02.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:02.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:02.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:02.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:02.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:02.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:02.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:02.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:02.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:02.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:02.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18528], [object_number=127, tag_tid=29, entry_tid=18527], [object_number=126, tag_tid=29, entry_tid=18526], [object_number=125, tag_tid=29, entry_tid=18525]]], state=connected] 2024-09-14T07:07:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:02.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:02.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:02.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:02.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:02.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:02.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18528], [object_number=127, tag_tid=29, entry_tid=18527], [object_number=126, tag_tid=29, entry_tid=18526], [object_number=125, tag_tid=29, entry_tid=18525]]], state=connected] 2024-09-14T07:07:02.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18528' '!=' 'object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18528' = 'object_number=124, tag_tid=29, entry_tid=18348' 2024-09-14T07:07:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:02.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:02.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:02.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:02.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18528 2024-09-14T07:07:02.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18528 2024-09-14T07:07:02.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:02.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:02.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:03.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:03.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:03.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:03.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:03.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:03.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:03.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:03.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:03.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:03.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:03.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:03.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:03.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:03.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:03.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:03.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:03.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:03.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:03.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:03.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:03.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:03.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:03.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:03.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:03.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:03.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:03.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=126, tag_tid=29, entry_tid=18670], [object_number=125, tag_tid=29, entry_tid=18669], [object_number=124, tag_tid=29, entry_tid=18668], [object_number=127, tag_tid=29, entry_tid=18667]]], state=connected] 2024-09-14T07:07:03.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:03.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:03.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:03.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:03.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:03.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:03.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:03.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:03.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=126, tag_tid=29, entry_tid=18670], [object_number=125, tag_tid=29, entry_tid=18669], [object_number=124, tag_tid=29, entry_tid=18668], [object_number=127, tag_tid=29, entry_tid=18667]]], state=connected] 2024-09-14T07:07:03.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=126, tag_tid=29, entry_tid=18670' '!=' 'object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=126, tag_tid=29, entry_tid=18670' = 'object_number=124, tag_tid=29, entry_tid=18528' 2024-09-14T07:07:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:03.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:03.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:03.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:03.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18670 2024-09-14T07:07:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18670 2024-09-14T07:07:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:03.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:03.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:03.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:03.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:03.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:03.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:03.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:03.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:03.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:03.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:03.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:03.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:03.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:03.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:03.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:03.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:04.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:04.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:04.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:04.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:04.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:04.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:04.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:04.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:04.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:04.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:04.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:04.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:04.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:04.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:04.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:04.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:04.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:04.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:04.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18808], [object_number=127, tag_tid=29, entry_tid=18807], [object_number=126, tag_tid=29, entry_tid=18806], [object_number=125, tag_tid=29, entry_tid=18805]]], state=connected] 2024-09-14T07:07:04.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:04.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:04.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:04.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=18808], [object_number=127, tag_tid=29, entry_tid=18807], [object_number=126, tag_tid=29, entry_tid=18806], [object_number=125, tag_tid=29, entry_tid=18805]]], state=connected] 2024-09-14T07:07:04.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:04.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:04.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18808' '!=' 'object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:04.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:04.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=18808' = 'object_number=126, tag_tid=29, entry_tid=18670' 2024-09-14T07:07:04.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:04.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:04.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:04.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:04.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:04.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:04.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:04.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:04.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:04.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:04.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:04.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:04.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:04.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18808 2024-09-14T07:07:04.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18808 2024-09-14T07:07:04.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:04.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:04.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:04.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:04.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:04.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:04.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:04.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:04.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:04.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:04.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:04.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:04.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:04.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:04.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:04.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:04.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:05.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:05.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:05.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:05.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=127, tag_tid=29, entry_tid=18943], [object_number=126, tag_tid=29, entry_tid=18942], [object_number=125, tag_tid=29, entry_tid=18941], [object_number=124, tag_tid=29, entry_tid=18940]]], state=connected] 2024-09-14T07:07:05.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=127, tag_tid=29, entry_tid=18943], [object_number=126, tag_tid=29, entry_tid=18942], [object_number=125, tag_tid=29, entry_tid=18941], [object_number=124, tag_tid=29, entry_tid=18940]]], state=connected] 2024-09-14T07:07:05.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=127, tag_tid=29, entry_tid=18943' '!=' 'object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=127, tag_tid=29, entry_tid=18943' = 'object_number=124, tag_tid=29, entry_tid=18808' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:05.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:05.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:05.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=18943 2024-09-14T07:07:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 18943 2024-09-14T07:07:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:05.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:05.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:05.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:05.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:05.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:05.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:05.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:05.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:05.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:05.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:05.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:05.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:05.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:05.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:05.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:05.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:05.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:05.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:05.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=126, tag_tid=29, entry_tid=19078], [object_number=125, tag_tid=29, entry_tid=19077], [object_number=124, tag_tid=29, entry_tid=19076], [object_number=127, tag_tid=29, entry_tid=19075]]], state=connected] 2024-09-14T07:07:05.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:05.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:05.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:05.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:06.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:06.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:06.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:06.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=126, tag_tid=29, entry_tid=19078], [object_number=125, tag_tid=29, entry_tid=19077], [object_number=124, tag_tid=29, entry_tid=19076], [object_number=127, tag_tid=29, entry_tid=19075]]], state=connected] 2024-09-14T07:07:06.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=126, tag_tid=29, entry_tid=19078' '!=' 'object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:06.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=126, tag_tid=29, entry_tid=19078' = 'object_number=127, tag_tid=29, entry_tid=18943' 2024-09-14T07:07:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:06.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:06.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:06.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:06.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:06.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:06.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19078 2024-09-14T07:07:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 19078 2024-09-14T07:07:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:06.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:06.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:06.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:06.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:06.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:06.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:06.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:06.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:06.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:06.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:06.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:06.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:06.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:06.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:06.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:06.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:06.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:06.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:06.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:06.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:06.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:06.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:06.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:06.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:06.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:06.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:06.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:06.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:06.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:06.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:06.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:06.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:06.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:06.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:06.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:06.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:06.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=19284], [object_number=127, tag_tid=29, entry_tid=19283], [object_number=126, tag_tid=29, entry_tid=19282], [object_number=125, tag_tid=29, entry_tid=19281]]], state=connected] 2024-09-14T07:07:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:06.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:06.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:06.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:06.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=124, tag_tid=29, entry_tid=19284], [object_number=127, tag_tid=29, entry_tid=19283], [object_number=126, tag_tid=29, entry_tid=19282], [object_number=125, tag_tid=29, entry_tid=19281]]], state=connected] 2024-09-14T07:07:06.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=19284' '!=' 'object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=124, tag_tid=29, entry_tid=19284' = 'object_number=126, tag_tid=29, entry_tid=19078' 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:06.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:06.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:06.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:06.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:06.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:06.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:06.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:06.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:06.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:06.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:06.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19284 2024-09-14T07:07:06.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 19284 2024-09-14T07:07:06.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:06.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:06.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:07.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:07.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:07.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:07.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:07.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:07.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:07.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:07.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:07.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:07.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:07.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:07.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:07.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:07.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:07.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:07.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:07.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:07.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:07.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:07.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:07.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:07.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:07.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:07.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:07.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:07.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:07.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:07.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:07.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:07.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:07.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:07.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:07.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:07.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:07.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:07.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:07.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:07.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:07.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:07.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:07.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:07.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:07.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:07.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:07.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:07.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:07.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:07.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:07.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:07.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:07.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:07.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:07.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:07.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:07.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:07.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:07.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:07.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=125, tag_tid=29, entry_tid=19369' '!=' 'object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:07.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:07:07.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=125, tag_tid=29, entry_tid=19369' = 'object_number=124, tag_tid=29, entry_tid=19284' 2024-09-14T07:07:07.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=30 2024-09-14T07:07:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:07:07.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=29 2024-09-14T07:07:07.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:07.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:07.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:07.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:07:07.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:07.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:07:07.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:07:07.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=19369 2024-09-14T07:07:07.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 30 = 29 -a 1 -le 19369 2024-09-14T07:07:07.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:07.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:07.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:07.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:07.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:07.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:07.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:07.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:07.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:07.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:07.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:07.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:07.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:07.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:07.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:07.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:07.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:07.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:07.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:07.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:08.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:08.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:08.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:08.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:08.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:08.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:08.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:08.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:08.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:08.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:08.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:08.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:08.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:08.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:08.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:08.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:08.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:08.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:08.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:08.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:08.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:08.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:08.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:08.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:08.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:08.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:08.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:08.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:08.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:08.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=125, tag_tid=29, entry_tid=19369' '!=' 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:08.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:08.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:07:08.852 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T07:07:08.852 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:07:08.852 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T07:07:08.852 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T07:07:08.853 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T07:07:08.855 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T07:07:08.856 INFO:tasks.rbd_mirror_thrash:waiting for 104 secs before thrashing 2024-09-14T07:07:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:08.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:08.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:08.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:08.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:08.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:08.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:08.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:08.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:08.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:08.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:08.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:08.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:08.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:08.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:09.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:09.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:09.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:09.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:09.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:09.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:09.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:09.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:09.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:09.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:09.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:09.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:09.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:09.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:09.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:09.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:09.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:09.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:09.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:09.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=125, tag_tid=29, entry_tid=19369' '!=' 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:09.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:09.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:10.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:10.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:10.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:10.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:10.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:10.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:10.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:10.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:10.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:10.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:10.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:10.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:10.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:10.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:10.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:10.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:10.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:10.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:10.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:10.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:10.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:10.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:10.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:10.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:10.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:10.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:10.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:10.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:10.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:10.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:10.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:10.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:10.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:10.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:10.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:10.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:10.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:10.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:10.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:10.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:10.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:10.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=29, entry_tid=19369], [object_number=124, tag_tid=29, entry_tid=19368], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:10.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:10.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:10.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=125, tag_tid=29, entry_tid=19369' '!=' 'object_number=125, tag_tid=29, entry_tid=19369' 2024-09-14T07:07:10.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:10.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:07:12.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:12.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:12.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:12.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:12.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:12.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:12.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:12.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:12.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:12.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:12.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:12.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:12.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:12.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:12.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:12.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:12.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:12.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:12.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:12.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:12.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:12.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:12.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:12.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:12.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:12.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:12.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:12.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=30, entry_tid=1], [object_number=124, tag_tid=30, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:12.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=30, entry_tid=1' -a 'object_number=125, tag_tid=30, entry_tid=1' = 'object_number=125, tag_tid=30, entry_tid=1' 2024-09-14T07:07:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:07:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:07:13.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2024-09-14T07:07:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:07:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:07:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:07:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 19 2024-09-14T07:07:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:07:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:04:43 2024 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:06:22 2024 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:07:14.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:07:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n snap9-clean ']' 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap9-clean 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9-clean 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:14.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:14.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:14.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:15.596 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:07:15.599+0000 7fea4b393640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:07:15.597 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:07:15.599+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2024-09-14T07:07:15.600 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:07:15.603+0000 7fea51ba0640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2024-09-14T07:07:15.816 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:07:15.815+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:07:15.902 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:07:15.903+0000 7f0de5b8b640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:07:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:16.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:16.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap10 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:17.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:17.816 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:07:17.815+0000 7fea4c395640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:07:18.260 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:07:18.263+0000 7fea4cb96640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:07:18.749 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.823358731958632) > weight (0.25) 2024-09-14T07:07:18.749 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6638615721813808) > weight (0.25) 2024-09-14T07:07:18.749 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T07:07:18.750 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T07:07:18.750 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T07:07:18.751 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:07:18.755+0000 7f7e5009f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 53559) UID: 1000 2024-09-14T07:07:18.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:18.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:19.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:19.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ clean_snap_name=snap10-clean 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror test snap10-clean 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10-clean 2024-09-14T07:07:19.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2024-09-14T07:07:19.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-14T07:07:19.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:19.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:07:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-14T07:07:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:20.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:20.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:07:20.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:07:20.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:07:20.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-14T07:07:20.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:07:20.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:20.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:20.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:07:20.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:07:20.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:20.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:07:20.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:07:20.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:20.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:20.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:20.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:20.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:20.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:20.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:20.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:20.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:20.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:20.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:20.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:20.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:20.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:07:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:21.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:21.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:21.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:21.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:21.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:21.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:21.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 1 2024-09-14T07:07:22.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:22.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:22.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:22.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:22.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:22.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:22.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:22.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:22.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:22.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:22.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:22.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:22.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:22.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:22.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:22.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:22.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:22.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:22.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:22.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 2 2024-09-14T07:07:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:24.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:24.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:24.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:24.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:24.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:24.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:24.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:24.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:24.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:24.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:24.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:24.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:24.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:24.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:25.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:25.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:25.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:25.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:25.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:25.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:25.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 4 2024-09-14T07:07:26.750 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T07:07:26.750 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:07:26.751 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T07:07:26.751 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T07:07:26.751 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T07:07:26.754 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T07:07:26.754 INFO:tasks.rbd_mirror_thrash:waiting for 80 secs before thrashing 2024-09-14T07:07:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:29.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:29.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:29.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:29.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:29.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:29.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:29.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:29.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:29.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:29.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:29.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:29.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:29.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:29.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:29.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:29.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:29.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:29.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:29.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:29.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:29.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:29.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:29.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:29.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:29.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:29.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:29.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:29.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:29.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:29.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:29.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:29.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:29.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:29.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:29.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:29.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:29.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:29.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:29.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T07:07:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:37.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:37.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:37.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:37.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:37.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:37.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:37.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:37.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:37.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:37.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:37.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:37.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:37.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:37.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:37.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:37.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:37.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:37.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:37.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:37.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:37.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:37.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:37.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:37.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:38.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:38.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:38.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:38.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:38.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:38.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:38.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:38.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:38.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:38.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:38.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:38.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:38.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:38.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:38.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:38.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:38.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T07:07:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:46.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:46.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:46.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:46.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:46.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:46.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:46.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:46.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:46.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:46.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:46.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:46.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:46.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:46.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:46.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:46.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:46.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:46.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:46.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:46.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:46.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:46.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:46.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:46.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:46.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:46.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:46.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:46.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:46.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:46.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:46.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:46.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:46.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:46.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T07:07:54.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:54.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:07:54.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:54.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:54.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:54.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:07:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:54.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-14T07:07:54.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:54.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:54.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-14T07:07:54.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:54.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:54.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:54.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:54.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-14T07:07:55.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:55.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:55.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-14T07:07:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:07:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:55.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:07:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:07:55.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:55.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:55.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:55.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:55.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-14T07:07:55.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:55.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:07:55.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:07:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:07:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:55.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:55.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:55.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-14T07:07:55.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:07:55.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:07:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:07:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:07:55.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror test 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n test ']' 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/test' 2024-09-14T07:07:55.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:07:55.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:07:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:07:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:07:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:07:55.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:55.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:55.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:07:55.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2024-09-14T07:07:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:55.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:55.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2024-09-14T07:07:55.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:07:55.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:07:55.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:07:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:07:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2024-09-14T07:07:55.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:07:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:07:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:07:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:07:55.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:55.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:55.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:07:55.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2024-09-14T07:07:55.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-14T07:07:55.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:07:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:07:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:07:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:07:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:07:55.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:07:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:07:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:07:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:07:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:07:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:07:55.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:07:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:07:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2024-09-14T07:07:56.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror test 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '' 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:07:56.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:56.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:56.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:56.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:56.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=31, entry_tid=1], [object_number=124, tag_tid=31, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=31, entry_tid=1], [object_number=124, tag_tid=31, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=125, tag_tid=31, entry_tid=1' 2024-09-14T07:07:56.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror test 2024-09-14T07:07:56.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:56.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror test '..*' 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-test.status 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-test.status 2024-09-14T07:07:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:07:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2024-09-14T07:07:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:56.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 31 2024-09-14T07:07:56.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 31 2024-09-14T07:07:56.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=125, tag_tid=31, entry_tid=1], [object_number=124, tag_tid=31, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=125, tag_tid=31, entry_tid=1], [object_number=124, tag_tid=31, entry_tid=0], [object_number=127, tag_tid=29, entry_tid=19367], [object_number=126, tag_tid=29, entry_tid=19366]]], state=connected] 2024-09-14T07:07:56.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.status 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=125, tag_tid=31, entry_tid=1' 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=125, tag_tid=31, entry_tid=1' -a 'object_number=125, tag_tid=31, entry_tid=1' = 'object_number=125, tag_tid=31, entry_tid=1' 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10-clean 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:07:56.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'test': 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 20 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 10a594c34cfd 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.10a594c34cfd 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:07:57.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 06:53:42 2024 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:07:14 2024 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:06:22 2024 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 10a594c34cfd 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5d62446-6732-43cb-bbad-b5dfd05689bd 2024-09-14T07:07:57.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:07:57.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:57.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 10 2024-09-14T07:07:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap1 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap1 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:57.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:57.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:57.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:57.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:58.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:07:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap1-clean 2024-09-14T07:07:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap1-clean 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap1-clean 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:07:59.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:07:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:00.384 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:00.382+0000 7fea4a391640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:00.384 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:00.382+0000 7fea4b393640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:00.384 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:08:00.382+0000 7f99dca15640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:08:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:00.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:01.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap2 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap2 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2 2024-09-14T07:08:01.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:01.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:01.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:01.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:01.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:01.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:03.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap2-clean 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap2-clean 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap2-clean 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:03.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:03.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:04.830 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:04.830+0000 7f0de4b89640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:08:04.831 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:04.830+0000 7f0de3386640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:08:04.831 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:04.830+0000 7f0de2b85640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2024-09-14T07:08:04.834 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:04.834+0000 7f0de5b8b640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:08:04.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:05.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:05.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap3 2024-09-14T07:08:05.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap3 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:06.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:06.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:07.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:07.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:07.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap3-clean 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap3-clean 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap3-clean 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:08.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:08.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:09.037 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:09.038+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:09.038 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:09.038+0000 7fea4db98640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:09.039 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:09.038+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:09.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:09.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap4 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap4 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:10.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:11.347 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:11.350+0000 7fea4ab92640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:11.347 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:11.350+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap4-clean 2024-09-14T07:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap4-clean 2024-09-14T07:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap4-clean 2024-09-14T07:08:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:12.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:12.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:13.689 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:13.694+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:13.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:13.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:14.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap5 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap5 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:15.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:15.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:16.239 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:16.242+0000 7fea4cb96640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:16.239 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:16.242+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:16.240 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:16.246+0000 7fea4a391640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:08:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:16.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:16.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap5-clean 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap5-clean 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap5-clean 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:16.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:16.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:17.679 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:17.682+0000 7fea4d397640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:18.294 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:18.298+0000 7f0de4388640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2024-09-14T07:08:18.294 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:18.298+0000 7f0de3386640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2024-09-14T07:08:18.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:18.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:19.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap6 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap6 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:19.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:19.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:19.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:20.457 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:20.462+0000 7fea4a391640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:20.897 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:08:20.902+0000 7f99de218640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:08:21.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:21.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:21.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:21.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap6-clean 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap6-clean 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap6-clean 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:21.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:22.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:22.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:23.081 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:23.082+0000 7fea4bb94640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:23.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:23.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:23.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap7 2024-09-14T07:08:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap7 2024-09-14T07:08:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:24.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7 2024-09-14T07:08:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:24.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:24.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:25.765 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:25.770+0000 7fea5039d640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:08:25.766 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:25.770+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:25.766 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:25.770+0000 7fea4ab92640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:08:26.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:26.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap7-clean 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap7-clean 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap7-clean 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:26.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:27.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:27.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:28.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:28.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:28.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:28.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap8 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap8 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:28.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:29.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:29.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:29.921 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:29.925+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:30.132 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:30.137+0000 7fea5039d640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:30.133 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:30.137+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:31.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:31.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:31.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap8-clean 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap8-clean 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap8-clean 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:32.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:32.708 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:08:32.713+0000 7f99de218640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:08:33.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:33.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:33.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap9 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap9 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:33.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:33.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:34.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:35.123 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:35.125+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:35.437 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:35.441+0000 7f0de6b8d640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:08:35.745 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:35.749+0000 7fea4e399640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:35.745 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:35.749+0000 7fea4d397640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:35.746 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:35.749+0000 7fea4c395640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:36.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:36.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:36.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap9-clean 2024-09-14T07:08:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap9-clean 2024-09-14T07:08:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:36.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap9-clean 2024-09-14T07:08:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:37.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:38.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:38.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap10 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap10 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:38.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:39.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:39.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:40.055 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.057+0000 7fea4cb96640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:40.056 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.057+0000 7fea4a391640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:40.058 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.061+0000 7fea5039d640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:40.258 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.261+0000 7fea4eb9a640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:40.370 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:08:40.373+0000 7f99dd216640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:08:40.480 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:08:40.485+0000 7f99dc214640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:08:40.485 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:08:40.489+0000 7f99e0a1d640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:08:40.577 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.581+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2024-09-14T07:08:40.577 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.581+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:40.581 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:40.585+0000 7f0de0380640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2024-09-14T07:08:40.584 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.581+0000 7fea4db98640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2024-09-14T07:08:40.584 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:40.581+0000 7fea4ab92640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:40.584 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:40.585+0000 7f0de1b83640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2024-09-14T07:08:40.586 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:08:40.589+0000 7f0de7b8f640 -1 bdev(0x5615c79c6c00 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2024-09-14T07:08:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:41.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:41.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap10-clean 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror test snap10-clean 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap10-clean 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:41.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export 2024-09-14T07:08:42.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:42.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:42.962 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:42.965+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:42.962 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:42.965+0000 7fea4eb9a640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:08:42.964 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:42.969+0000 7fea4a391640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2024-09-14T07:08:43.263 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:43.265+0000 7fea4c395640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:43.265 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:43.269+0000 7fea4d397640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:43.267 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:43.269+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:43.276 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:08:43.277+0000 7fea5139f640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:08:43.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-14T07:08:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:43.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-test.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-test.export 2024-09-14T07:08:43.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:08:43.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 10 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap1 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap1 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap1 2024-09-14T07:08:43.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2024-09-14T07:08:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap1-clean 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap1-clean 2024-09-14T07:08:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2024-09-14T07:08:45.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap2 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap2 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap2 2024-09-14T07:08:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2024-09-14T07:08:46.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap2-clean 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap2-clean 2024-09-14T07:08:46.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2024-09-14T07:08:46.755 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8394811108633757) > weight (0.25) 2024-09-14T07:08:46.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.860903598050477) > weight (0.25) 2024-09-14T07:08:46.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9423001112178278) > weight (0.25) 2024-09-14T07:08:46.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9600301584198464) > weight (0.25) 2024-09-14T07:08:46.756 INFO:tasks.rbd_mirror_thrash:waiting for 72 secs before thrashing 2024-09-14T07:08:47.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap3 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap3 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:47.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap3 2024-09-14T07:08:47.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2024-09-14T07:08:48.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:48.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap3-clean 2024-09-14T07:08:48.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2024-09-14T07:08:48.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:48.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:48.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:48.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap3-clean 2024-09-14T07:08:48.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2024-09-14T07:08:49.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap4 2024-09-14T07:08:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap4 2024-09-14T07:08:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap4 2024-09-14T07:08:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2024-09-14T07:08:50.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:50.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap4-clean 2024-09-14T07:08:50.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2024-09-14T07:08:50.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:50.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:50.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:50.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap4-clean 2024-09-14T07:08:50.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2024-09-14T07:08:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap5 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap5 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap5 2024-09-14T07:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2024-09-14T07:08:52.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap5-clean 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap5-clean 2024-09-14T07:08:52.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2024-09-14T07:08:52.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.46885365860404005) > weight (0.25) 2024-09-14T07:08:52.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6939018403890937) > weight (0.25) 2024-09-14T07:08:52.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9885969063159402) > weight (0.25) 2024-09-14T07:08:52.858 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8288297929657227) > weight (0.25) 2024-09-14T07:08:52.858 INFO:tasks.rbd_mirror_thrash:waiting for 82 secs before thrashing 2024-09-14T07:08:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:53.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:53.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap6 2024-09-14T07:08:53.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap6 2024-09-14T07:08:53.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:53.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:53.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:53.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap6 2024-09-14T07:08:53.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2024-09-14T07:08:54.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap6-clean 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap6-clean 2024-09-14T07:08:54.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2024-09-14T07:08:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap7 2024-09-14T07:08:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap7 2024-09-14T07:08:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:55.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap7 2024-09-14T07:08:55.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2024-09-14T07:08:56.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:56.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap7-clean 2024-09-14T07:08:56.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2024-09-14T07:08:56.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:56.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap7-clean 2024-09-14T07:08:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2024-09-14T07:08:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap8 2024-09-14T07:08:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap8 2024-09-14T07:08:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:57.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:57.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap8 2024-09-14T07:08:57.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2024-09-14T07:08:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap8-clean 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap8-clean 2024-09-14T07:08:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2024-09-14T07:08:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap9 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap9 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap9 2024-09-14T07:08:59.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2024-09-14T07:09:00.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap9-clean 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap9-clean 2024-09-14T07:09:00.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2024-09-14T07:09:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:09:01.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:09:01.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap10 2024-09-14T07:09:01.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap10 2024-09-14T07:09:01.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:01.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:01.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:01.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap10 2024-09-14T07:09:01.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2024-09-14T07:09:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:09:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap10-clean 2024-09-14T07:09:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2024-09-14T07:09:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap10-clean 2024-09-14T07:09:02.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2024-09-14T07:09:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:09:03.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror test 2024-09-14T07:09:03.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:03.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:03.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:03.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:09:03.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:09:03.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror test 2024-09-14T07:09:03.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:03.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:03.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:03.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2024-09-14T07:09:03.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/test 2024-09-14T07:09:03.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:09:03.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:03.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror test deleted 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror test 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-09-14T07:09:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:09:04.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image test: (2) No such file or directory 2024-09-14T07:09:04.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:09:04.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:09:04.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:09:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror test deleted '' 2024-09-14T07:09:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:09:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=test 2024-09-14T07:09:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:09:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:09:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:09:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:09:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror test 2024-09-14T07:09:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:09:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:09:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=test 2024-09-14T07:09:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-09-14T07:09:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:09:04.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image test: (2) No such file or directory 2024-09-14T07:09:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:09:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:09:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:09:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: create many images' 2024-09-14T07:09:04.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T07:09:04.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T07:09:04.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 07:09:04 TEST: create many images 2024-09-14T07:09:04.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 07:09:04 TEST: create many images 2024-09-14T07:09:04.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ snap_name=snap 2024-09-14T07:09:04.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 50 2024-09-14T07:09:04.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:09:04.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_1 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 journal 128M 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_1 128M 2024-09-14T07:09:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_1 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_1 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:04.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:09:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:09:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:09:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:09:04.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:09:05.383 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x55921002de10 2024-09-14T07:09:05.383 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x559210517110 2024-09-14T07:09:05.386 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x559211052c30 2024-09-14T07:09:05.386 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x5592115e5c70 2024-09-14T07:09:05.386 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:09:05.387 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:09:05.387 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:09:05.385+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:09:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 78510080 bytes 2024-09-14T07:09:09.071 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:09:09.074+0000 7f7c359f7640 -1 reset not still connected to 0x55ba92131930 2024-09-14T07:09:09.071 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:09:09.074+0000 7f7c359f7640 -1 reset not still connected to 0x55bab230ba00 2024-09-14T07:09:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 130290688 bytes 2024-09-14T07:09:14.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:09:14.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_2 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 journal 128M 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:09:14.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_2 128M 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_2 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_2 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:14.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:09:14.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:09:14.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:09:14.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:09:14.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:09:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 16738304 bytes 2024-09-14T07:09:19.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 56585216 bytes 2024-09-14T07:09:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:09:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:09:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:09:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_3 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 journal 128M 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_3 128M 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_3 2024-09-14T07:09:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:09:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:09:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:09:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:09:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:09:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_3 2024-09-14T07:09:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:09:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:09:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:09:23.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:09:23.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:09:24.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 19607552 bytes 2024-09-14T07:09:27.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 79992832 bytes 2024-09-14T07:09:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 109509632 bytes 2024-09-14T07:09:31.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 167776256 bytes 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_4 2024-09-14T07:09:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 journal 128M 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_4 128M 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_4 2024-09-14T07:09:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_4 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:09:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:09:33.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:09:33.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:09:33.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:09:36.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 43234304 bytes 2024-09-14T07:09:38.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 82644992 bytes 2024-09-14T07:09:42.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 156570624 bytes 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_5 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 journal 128M 2024-09-14T07:09:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_5 128M 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:09:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:09:45.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:09:45.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:09:45.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:45.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_5 2024-09-14T07:09:45.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:09:45.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_5 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:09:45.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:09:45.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=7 2024-09-14T07:09:45.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:09:45.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:09:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 51726336 bytes 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_6 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 journal 128M 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:52.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_6 128M 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:09:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:09:52.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:09:52.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:09:52.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:09:52.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_6 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_6 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:09:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:09:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:09:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=13 2024-09-14T07:09:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:09:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:09:54.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 35225600 bytes 2024-09-14T07:09:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 92790784 bytes 2024-09-14T07:09:58.215 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:09:58.217+0000 7f7dde471640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 1 2024-09-14T07:09:58.310 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:09:58.312+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca17cb60 2024-09-14T07:09:58.310 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:09:58.312+0000 7f0df4fca640 -1 reset not still connected to 0x5615f01696c0 2024-09-14T07:09:58.757 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:09:58.757 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:09:58.758 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-14T07:09:58.758 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:09:58.761+0000 7fc706c78640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 78077) UID: 1000 2024-09-14T07:09:59.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 109490176 bytes 2024-09-14T07:10:00.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:10:00.505+0000 7fc6ffc6a640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55c2fe92c000 handle_process_entry_safe: failed to commit journal event: (108) Cannot send after transport endpoint shutdown 2024-09-14T07:10:00.504 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:10:00.505+0000 7fc6ffc6a640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55c2fe92c000 handle_replay_complete: replay encountered an error: (108) Cannot send after transport endpoint shutdown 2024-09-14T07:10:02.001 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:02.003+0000 7f99ede58640 -1 reset not still connected to 0x559211151860 2024-09-14T07:10:02.001 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:02.003+0000 7f99ede58640 -1 reset not still connected to 0x55921197e750 2024-09-14T07:10:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 189924352 bytes 2024-09-14T07:10:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 214349824 bytes 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_7 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 journal 128M 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_7 128M 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_7 2024-09-14T07:10:05.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:05.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:05.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_7 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:10:05.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:05.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:06.213 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:06.215+0000 7f99ede58640 -1 reset not still connected to 0x5592118cef70 2024-09-14T07:10:06.213 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:06.215+0000 7f99ede58640 -1 reset not still connected to 0x5592118cf040 2024-09-14T07:10:06.215 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:06.215+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:10:06.758 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:10:06.759 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:10:06.759 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:10:06.759 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:10:06.760 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:10:06.762 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:10:06.762 INFO:tasks.rbd_mirror_thrash:waiting for 89 secs before thrashing 2024-09-14T07:10:09.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 105929728 bytes 2024-09-14T07:10:11.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 132553728 bytes 2024-09-14T07:10:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 138209280 bytes 2024-09-14T07:10:13.433 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:13.435+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:10:13.433 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:13.435+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:10:13.434 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:13.435+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:10:14.859 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T07:10:14.859 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T07:10:14.860 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T07:10:14.860 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:10:14.861+0000 7fb48935d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 30481) UID: 1000 2024-09-14T07:10:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_8 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 journal 128M 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_8 128M 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:15.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:15.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_8 2024-09-14T07:10:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_8 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:10:15.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:15.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:10:15.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:15.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 94583808 bytes 2024-09-14T07:10:21.860 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T07:10:21.860 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:10:21.861 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T07:10:21.861 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T07:10:21.861 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T07:10:21.863 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T07:10:21.864 INFO:tasks.rbd_mirror_thrash:waiting for 115 secs before thrashing 2024-09-14T07:10:23.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 118955008 bytes 2024-09-14T07:10:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_9 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 journal 128M 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_9 128M 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_9 2024-09-14T07:10:24.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:24.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:24.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:24.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:24.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:24.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_9 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:10:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:24.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:28.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 47006720 bytes 2024-09-14T07:10:30.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 105241600 bytes 2024-09-14T07:10:33.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 136806400 bytes 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_10 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 journal 128M 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_10 128M 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_10 2024-09-14T07:10:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:33.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_10 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:10:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:33.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:10:33.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:33.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:36.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 39147520 bytes 2024-09-14T07:10:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 39739392 bytes 2024-09-14T07:10:38.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 40368128 bytes 2024-09-14T07:10:39.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 54276096 bytes 2024-09-14T07:10:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 82950144 bytes 2024-09-14T07:10:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:43.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_11 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 journal 128M 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_11 128M 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_11 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_11 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T07:10:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:48.147 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:10:48.152+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_12 2024-09-14T07:10:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 journal 128M 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_12 128M 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_12 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_12 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:10:49.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:49.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:10:49.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:49.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:49.365 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:10:49.366+0000 7fea5efdb640 -1 reset not still connected to 0x55844076a8f0 2024-09-14T07:10:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 32424960 bytes 2024-09-14T07:10:51.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:10:51.603+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:10:53.148 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:10:53.151+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:10:53.265 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:10:53.267+0000 7f7c359f7640 -1 reset not still connected to 0x55bab230b5f0 2024-09-14T07:10:53.267 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:10:53.271+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616e680 2024-09-14T07:10:54.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 77216768 bytes 2024-09-14T07:10:56.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:10:56.607+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:10:57.332 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:57.334+0000 7f99ede58640 -1 reset not still connected to 0x559211560820 2024-09-14T07:10:57.332 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:57.334+0000 7f99ede58640 -1 reset not still connected to 0x5592115ecea0 2024-09-14T07:10:57.332 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:57.334+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:10:57.333 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:57.334+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b6a90 2024-09-14T07:10:58.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 120263680 bytes 2024-09-14T07:10:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:10:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:10:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:10:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:10:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:10:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_13 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 journal 128M 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_13 128M 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:10:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_13 2024-09-14T07:10:58.149 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:10:58.151+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:10:58.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:10:58.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_13 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:10:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:10:58.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=14 2024-09-14T07:10:58.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:10:58.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:10:58.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 607232 bytes 2024-09-14T07:10:58.608 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:58.610+0000 7f99ede58640 -1 reset not still connected to 0x5592118cf040 2024-09-14T07:10:58.609 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:58.614+0000 7f99ede58640 -1 reset not still connected to 0x5592168c55f0 2024-09-14T07:10:58.609 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:10:58.614+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:10:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 24161280 bytes 2024-09-14T07:11:01.603 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:01.607+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:11:03.150 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:03.151+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:11:03.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 59983872 bytes 2024-09-14T07:11:03.573 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:11:03.575+0000 7f7c359f7640 -1 reset not still connected to 0x55ba945b60d0 2024-09-14T07:11:03.573 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:11:03.575+0000 7f7c359f7640 -1 reset not still connected to 0x55ba9e548d00 2024-09-14T07:11:05.411 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:05.415+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f680 finish: resending after timeout 2024-09-14T07:11:05.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 95584256 bytes 2024-09-14T07:11:06.604 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:06.607+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:11:07.246 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:11:07.251+0000 7f7c359f7640 -1 reset not still connected to 0x55ba92131790 2024-09-14T07:11:07.247 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:11:07.251+0000 7f7c359f7640 -1 reset not still connected to 0x55ba92a852b0 2024-09-14T07:11:07.247 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:11:07.251+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616e4e0 2024-09-14T07:11:08.150 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:08.155+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:11:10.664 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:10.665+0000 7f99ede58640 -1 reset not still connected to 0x559211560820 2024-09-14T07:11:10.672 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:10.673+0000 7f99ede58640 -1 reset not still connected to 0x5592115ecea0 2024-09-14T07:11:10.672 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:10.673+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:11:10.672 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:10.673+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b6a90 2024-09-14T07:11:11.552 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:11.553+0000 7fea5efdb640 -1 reset not still connected to 0x558446302000 2024-09-14T07:11:11.553 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:11.553+0000 7fea5efdb640 -1 reset not still connected to 0x558446302a90 2024-09-14T07:11:11.553 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:11.553+0000 7fea5efdb640 -1 reset not still connected to 0x55845f031110 2024-09-14T07:11:11.604 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:11.607+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:11:11.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 180338688 bytes 2024-09-14T07:11:12.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:11:12.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_14 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 journal 128M 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:11:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_14 128M 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_14 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_14 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:11:12.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:11:12.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=6 2024-09-14T07:11:12.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:11:12.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:11:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 11831296 bytes 2024-09-14T07:11:13.152 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:13.155+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:11:13.661 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:13.665+0000 7f99ede58640 -1 reset not still connected to 0x5592115ec9c0 2024-09-14T07:11:13.661 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:13.665+0000 7f99ede58640 -1 reset not still connected to 0x5592118cef70 2024-09-14T07:11:13.661 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:13.665+0000 7f99ede58640 -1 reset not still connected to 0x5592118cf040 2024-09-14T07:11:15.759 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.761+0000 7fea5efdb640 -1 reset not still connected to 0x558440d98340 2024-09-14T07:11:15.767 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x558440da96c0 2024-09-14T07:11:15.768 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03930 2024-09-14T07:11:15.768 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x55843f0dbba0 2024-09-14T07:11:15.768 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x55843f256680 2024-09-14T07:11:15.769 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x55843f72dc70 2024-09-14T07:11:15.769 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x55844078e680 2024-09-14T07:11:15.769 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:11:15.769 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:11:15.769 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:15.765+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:11:17.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 67945472 bytes 2024-09-14T07:11:18.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:11:18.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:11:18.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:11:18.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:11:18.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:11:18.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_15 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 journal 128M 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_15 128M 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_15 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_15 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:11:18.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:11:18.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:11:18.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:11:18.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:11:19.387 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:19.391+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:11:19.809 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:19.813+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:11:19.809 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:19.813+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:11:19.809 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:19.813+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:11:21.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 30883840 bytes 2024-09-14T07:11:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 83299328 bytes 2024-09-14T07:11:24.388 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:24.390+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:11:24.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:24.810+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:11:24.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:24.814+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:11:25.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 95959040 bytes 2024-09-14T07:11:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:11:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:11:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:11:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:11:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_16 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 journal 128M 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_16 128M 2024-09-14T07:11:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:27.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_16 2024-09-14T07:11:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_16 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:11:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:11:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=6 2024-09-14T07:11:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:11:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:11:28.366 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:28.369+0000 7f99ede58640 -1 reset not still connected to 0x559211151860 2024-09-14T07:11:28.367 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:11:28.369+0000 7f99ede58640 -1 reset not still connected to 0x55921197e750 2024-09-14T07:11:29.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 28354560 bytes 2024-09-14T07:11:29.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:29.810+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341e3c0 finish: resending after timeout 2024-09-14T07:11:29.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:29.814+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:11:32.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:32.266+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:11:32.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 53399552 bytes 2024-09-14T07:11:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:11:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:11:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:11:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_17 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 journal 128M 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_17 128M 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:33.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_17 2024-09-14T07:11:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:11:33.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_17 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:11:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:11:33.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:11:33.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:11:33.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:11:35.201 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:35.202+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aada40 finish: resending after timeout 2024-09-14T07:11:35.203 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:35.206+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aaca80 finish: resending after timeout 2024-09-14T07:11:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 27091968 bytes 2024-09-14T07:11:35.763 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8012057624467385) > weight (0.25) 2024-09-14T07:11:35.763 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9501542731811797) > weight (0.25) 2024-09-14T07:11:35.763 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3320167728087602) > weight (0.25) 2024-09-14T07:11:35.763 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.3160303590723217) > weight (0.25) 2024-09-14T07:11:35.764 INFO:tasks.rbd_mirror_thrash:waiting for 65 secs before thrashing 2024-09-14T07:11:38.146 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:38.146+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:11:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 91296768 bytes 2024-09-14T07:11:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 145879040 bytes 2024-09-14T07:11:43.146 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:43.146+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:11:44.883 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:44.886+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aada40 finish: resending after timeout 2024-09-14T07:11:45.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:11:45.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:11:45.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:11:45.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:11:45.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_18 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 journal 128M 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_18 128M 2024-09-14T07:11:45.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_18 2024-09-14T07:11:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:11:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:11:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:11:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:11:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:11:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_18 2024-09-14T07:11:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:11:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:11:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:11:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:11:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:11:47.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 27348992 bytes 2024-09-14T07:11:48.949 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:11:48.952+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:11:50.171 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:11:50.174+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:11:50.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 46371840 bytes 2024-09-14T07:11:54.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 103165952 bytes 2024-09-14T07:11:54.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 103756800 bytes 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_19 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 journal 128M 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_19 128M 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:11:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:11:55.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:11:55.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:11:55.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:11:55.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_19 2024-09-14T07:11:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:11:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:11:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:11:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:11:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:11:55.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_19 2024-09-14T07:11:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:11:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:11:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:11:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:11:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=7 2024-09-14T07:11:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:11:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:11:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 40479744 bytes 2024-09-14T07:12:01.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 60460032 bytes 2024-09-14T07:12:02.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 76012544 bytes 2024-09-14T07:12:02.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 77374464 bytes 2024-09-14T07:12:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:12:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:12:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:12:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:12:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:12:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_20 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 journal 128M 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_20 128M 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:02.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_20 2024-09-14T07:12:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:12:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:12:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:12:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:12:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_20 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:12:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:12:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:12:05.068 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:05.068+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:12:05.074 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:05.072+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:12:05.074 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:05.072+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:12:05.110 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:05.112+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685110 2024-09-14T07:12:05.111 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:05.112+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf1d8ea0 2024-09-14T07:12:06.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 41822208 bytes 2024-09-14T07:12:07.667 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:12:07.665+0000 7f7c359f7640 -1 reset not still connected to 0x55ba945b60d0 2024-09-14T07:12:07.667 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:12:07.665+0000 7f7c359f7640 -1 reset not still connected to 0x55ba9e548d00 2024-09-14T07:12:08.121 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:08.120+0000 7fea5efdb640 -1 reset not still connected to 0x55844078e680 2024-09-14T07:12:08.121 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:08.120+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:12:08.121 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:08.120+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:12:08.121 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:08.120+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:12:10.297 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:10.300+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca17cb60 2024-09-14T07:12:10.297 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:10.300+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca17dd40 2024-09-14T07:12:10.298 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:10.300+0000 7f0df4fca640 -1 reset not still connected to 0x5615f01696c0 2024-09-14T07:12:11.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 106765312 bytes 2024-09-14T07:12:13.920 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.920+0000 7f99ede58640 -1 reset not still connected to 0x559210517110 2024-09-14T07:12:13.921 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.920+0000 7f99ede58640 -1 reset not still connected to 0x559210932000 2024-09-14T07:12:13.923 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.924+0000 7f99ede58640 -1 reset not still connected to 0x5592115e5c70 2024-09-14T07:12:13.923 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.924+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:12:13.923 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.924+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:12:13.923 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.924+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:12:13.923 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:13.924+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:12:14.157 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:12:14.157+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aada40 finish: resending after timeout 2024-09-14T07:12:14.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_21 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 journal 128M 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:12:14.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_21 128M 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:12:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:14.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_21 2024-09-14T07:12:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:12:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_21 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:12:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:12:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=6 2024-09-14T07:12:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:12:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:12:16.865 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6987248439813469) > weight (0.25) 2024-09-14T07:12:16.865 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5996087181888992) > weight (0.25) 2024-09-14T07:12:16.865 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:12:16.865 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:12:16.866 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T07:12:16.866 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:12:16.866 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:12:16.866 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:12:16.869+0000 7fb1560b7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 77977) UID: 1000 2024-09-14T07:12:17.010 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:12:17.009+0000 7fb3f8d94640 -1 reset not still connected to 0x5614be399860 2024-09-14T07:12:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 39572480 bytes 2024-09-14T07:12:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:12:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:12:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:12:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:12:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:12:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_22 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 journal 128M 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_22 128M 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:21.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_22 2024-09-14T07:12:21.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:12:21.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:21.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_22 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:12:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:12:21.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:12:21.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:12:21.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:12:22.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 13506560 bytes 2024-09-14T07:12:22.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 14118912 bytes 2024-09-14T07:12:22.968 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:12:22.968 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:12:22.968 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:12:22.971 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:12:22.971 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-09-14T07:12:23.153 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:23.155+0000 7f99ede58640 -1 reset not still connected to 0x5592115ecea0 2024-09-14T07:12:23.154 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:23.155+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:12:23.154 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:23.155+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b6a90 2024-09-14T07:12:25.052 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:25.051+0000 7f99ede58640 -1 reset not still connected to 0x5592168c55f0 2024-09-14T07:12:25.053 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:25.051+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:12:25.053 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:25.051+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:12:25.055 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:12:25.055+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:12:25.127 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:25.127+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69ca90 2024-09-14T07:12:25.127 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:25.127+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:12:25.128 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:25.127+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:12:25.430 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:25.427+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685e10 2024-09-14T07:12:25.430 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:25.427+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf164ea0 2024-09-14T07:12:28.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 100807680 bytes 2024-09-14T07:12:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 121945088 bytes 2024-09-14T07:12:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:12:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:12:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:12:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:12:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:12:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_23 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 journal 128M 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_23 128M 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_23 2024-09-14T07:12:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:12:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:12:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:12:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_23 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:12:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:12:31.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:12:31.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:12:33.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 32594944 bytes 2024-09-14T07:12:36.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 53372928 bytes 2024-09-14T07:12:37.654 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:12:37.652+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:12:38.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 92781568 bytes 2024-09-14T07:12:38.164 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:38.163+0000 7fea5efdb640 -1 reset not still connected to 0x558446302000 2024-09-14T07:12:38.164 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:38.163+0000 7fea5efdb640 -1 reset not still connected to 0x558446302a90 2024-09-14T07:12:38.166 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:38.167+0000 7fea5efdb640 -1 reset not still connected to 0x55845f031110 2024-09-14T07:12:39.622 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:12:39.624+0000 7fb3f8d94640 -1 reset not still connected to 0x5614a8f52a90 2024-09-14T07:12:40.764 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2851215275533149) > weight (0.25) 2024-09-14T07:12:40.764 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4104426205684165) > weight (0.25) 2024-09-14T07:12:40.764 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.41318677339989796) > weight (0.25) 2024-09-14T07:12:40.765 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6457283193219521) > weight (0.25) 2024-09-14T07:12:40.765 INFO:tasks.rbd_mirror_thrash:waiting for 92 secs before thrashing 2024-09-14T07:12:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 137842688 bytes 2024-09-14T07:12:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:12:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_24 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 journal 128M 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_24 128M 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:12:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_24 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_24 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:12:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:12:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:12:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:12:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:12:44.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 13754368 bytes 2024-09-14T07:12:47.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 50503680 bytes 2024-09-14T07:12:48.205 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:48.203+0000 7f0df4fca640 -1 reset not still connected to 0x5615cb53d040 2024-09-14T07:12:48.205 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:48.207+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685110 2024-09-14T07:12:49.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 97457152 bytes 2024-09-14T07:12:51.276 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:12:51.280+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:12:52.895 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:12:52.900+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616f790 2024-09-14T07:12:54.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 162806784 bytes 2024-09-14T07:12:55.026 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:55.027+0000 7fea5efdb640 -1 reset not still connected to 0x55845f031110 2024-09-14T07:12:55.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 163415040 bytes 2024-09-14T07:12:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:12:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_25 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 journal 128M 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_25 128M 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:12:55.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_25 2024-09-14T07:12:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:12:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:12:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:12:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:12:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:12:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_25 2024-09-14T07:12:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:12:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:12:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:12:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:12:55.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:12:55.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:12:55.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:12:56.279 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:12:56.280+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:12:56.543 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:56.543+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685110 2024-09-14T07:12:56.546 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:12:56.547+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf1d8ea0 2024-09-14T07:12:58.042 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:58.038+0000 7fea5efdb640 -1 reset not still connected to 0x5584448dfd40 2024-09-14T07:12:58.042 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:58.038+0000 7fea5efdb640 -1 reset not still connected to 0x558446184d00 2024-09-14T07:12:58.042 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:58.038+0000 7fea5efdb640 -1 reset not still connected to 0x558446302000 2024-09-14T07:12:58.042 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:58.038+0000 7fea5efdb640 -1 reset not still connected to 0x558446302a90 2024-09-14T07:12:58.043 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:58.038+0000 7fea5efdb640 -1 reset not still connected to 0x55845f031110 2024-09-14T07:12:58.275 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:12:58.274+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:12:59.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 49104896 bytes 2024-09-14T07:13:02.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 89455616 bytes 2024-09-14T07:13:02.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 112605184 bytes 2024-09-14T07:13:03.830 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:03.830+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b7ee0 2024-09-14T07:13:04.083 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:04.082+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b7ee0 2024-09-14T07:13:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 159224832 bytes 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_26 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 journal 128M 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:13:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_26 128M 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:13:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_26 2024-09-14T07:13:07.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:13:07.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:07.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:13:07.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:13:07.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:13:07.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_26 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:13:07.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:13:07.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:13:09.051 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:13:09.051+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341f200 finish: resending after timeout 2024-09-14T07:13:09.774 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:09.774+0000 7f99ede58640 -1 reset not still connected to 0x5592115e5c70 2024-09-14T07:13:09.774 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:09.774+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:13:09.774 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:09.774+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:13:09.776 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:09.774+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:13:09.776 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:09.774+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:13:10.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:10.378+0000 7fea5efdb640 -1 reset not still connected to 0x558446302000 2024-09-14T07:13:10.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:10.378+0000 7fea5efdb640 -1 reset not still connected to 0x558446302a90 2024-09-14T07:13:10.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:10.378+0000 7fea5efdb640 -1 reset not still connected to 0x55845f031110 2024-09-14T07:13:12.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 55020544 bytes 2024-09-14T07:13:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 55922688 bytes 2024-09-14T07:13:14.191 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:14.186+0000 7f99ede58640 -1 reset not still connected to 0x55921002c750 2024-09-14T07:13:14.191 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:14.186+0000 7f99ede58640 -1 reset not still connected to 0x559211560820 2024-09-14T07:13:14.192 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:14.186+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:13:14.297 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:13:14.299+0000 7fb3e7150640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:13:14.306 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:13:14.307+0000 7fb3e514c640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:13:14.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 75848704 bytes 2024-09-14T07:13:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 113878016 bytes 2024-09-14T07:13:16.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_27 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 journal 128M 2024-09-14T07:13:16.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_27 128M 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:16.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_27 2024-09-14T07:13:16.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:13:16.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:16.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:13:16.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:13:16.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:13:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:13:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_27 2024-09-14T07:13:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:13:16.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:13:16.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:13:16.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:13:16.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:13:19.866 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.870+0000 7fea5efdb640 -1 reset not still connected to 0x55843d6f2340 2024-09-14T07:13:19.867 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.870+0000 7fea5efdb640 -1 reset not still connected to 0x55843d8aa750 2024-09-14T07:13:19.867 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.870+0000 7fea5efdb640 -1 reset not still connected to 0x55843f2569c0 2024-09-14T07:13:19.867 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.870+0000 7fea5efdb640 -1 reset not still connected to 0x55844076a8f0 2024-09-14T07:13:19.867 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.870+0000 7fea5efdb640 -1 reset not still connected to 0x558440d935f0 2024-09-14T07:13:19.911 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.914+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:13:19.919 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.922+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:13:19.919 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.922+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:13:19.985 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.986+0000 7fea5efdb640 -1 reset not still connected to 0x55843d6f2340 2024-09-14T07:13:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.986+0000 7fea5efdb640 -1 reset not still connected to 0x55843d8aa750 2024-09-14T07:13:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.986+0000 7fea5efdb640 -1 reset not still connected to 0x55843f2569c0 2024-09-14T07:13:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.986+0000 7fea5efdb640 -1 reset not still connected to 0x55844076a8f0 2024-09-14T07:13:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.986+0000 7fea5efdb640 -1 reset not still connected to 0x558440d935f0 2024-09-14T07:13:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:19.986+0000 7fea5efdb640 -1 reset not still connected to 0x558446185ba0 2024-09-14T07:13:20.650 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:20.650+0000 7fea5efdb640 -1 reset not still connected to 0x558440da96c0 2024-09-14T07:13:20.650 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:20.650+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03930 2024-09-14T07:13:20.651 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:20.650+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:13:20.651 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:20.654+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:13:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 55083008 bytes 2024-09-14T07:13:21.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 55684096 bytes 2024-09-14T07:13:22.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 71213056 bytes 2024-09-14T07:13:24.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 89796608 bytes 2024-09-14T07:13:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 100188160 bytes 2024-09-14T07:13:25.432 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:13:25.435+0000 7f7c265b8640 -1 bdev(0x55ba904d4c00 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 5 2024-09-14T07:13:25.433 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:13:25.435+0000 7f7c285bc640 -1 bdev(0x55ba904d4c00 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 7 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_28 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 journal 128M 2024-09-14T07:13:25.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_28 128M 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:25.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:25.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_28 2024-09-14T07:13:25.776 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:25.778+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_28 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:13:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:13:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=13 2024-09-14T07:13:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:13:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:13:25.933 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:25.934+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:13:25.933 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:25.934+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:13:25.933 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:25.934+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:13:26.501 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:13:26.502+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf164ea0 2024-09-14T07:13:26.502 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:13:26.502+0000 7f0df4fca640 -1 reset not still connected to 0x5615f2894dd0 2024-09-14T07:13:26.972 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T07:13:26.973 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T07:13:26.973 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T07:13:26.974 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:13:26.975+0000 7fad4966f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 87130) UID: 1000 2024-09-14T07:13:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 16488448 bytes 2024-09-14T07:13:28.209 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:28.209+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:13:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 76544000 bytes 2024-09-14T07:13:32.809 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:32.809+0000 7f99ede58640 -1 reset not still connected to 0x5592109324e0 2024-09-14T07:13:32.809 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:32.809+0000 7f99ede58640 -1 reset not still connected to 0x559211151860 2024-09-14T07:13:32.809 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:32.809+0000 7f99ede58640 -1 reset not still connected to 0x55921197e750 2024-09-14T07:13:32.812 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:32.813+0000 7f99ede58640 -1 reset not still connected to 0x559218b04ea0 2024-09-14T07:13:32.812 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:32.813+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2680 2024-09-14T07:13:32.973 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T07:13:32.974 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:13:32.974 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T07:13:32.974 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T07:13:32.974 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T07:13:32.977 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T07:13:32.977 INFO:tasks.rbd_mirror_thrash:waiting for 111 secs before thrashing 2024-09-14T07:13:33.514 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:33.517+0000 7fea5efdb640 -1 reset not still connected to 0x558446185ba0 2024-09-14T07:13:33.514 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:33.517+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff4b5f0 2024-09-14T07:13:35.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 135003136 bytes 2024-09-14T07:13:38.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:13:38.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_29 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 journal 128M 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:13:38.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_29 128M 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:38.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_29 2024-09-14T07:13:38.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:13:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:13:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:13:38.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:13:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:13:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_29 2024-09-14T07:13:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:13:38.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:13:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:13:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:13:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:13:40.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 17995776 bytes 2024-09-14T07:13:42.781 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:13:42.781+0000 7f0df4fca640 -1 reset not still connected to 0x5615cbb05ee0 2024-09-14T07:13:42.781 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:13:42.781+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685110 2024-09-14T07:13:42.781 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:13:42.781+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf1d8ea0 2024-09-14T07:13:43.278 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:43.277+0000 7fea5efdb640 -1 reset not still connected to 0x558440d935f0 2024-09-14T07:13:43.280 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:43.281+0000 7fea5efdb640 -1 reset not still connected to 0x558446185ba0 2024-09-14T07:13:43.280 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:43.281+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff4b5f0 2024-09-14T07:13:43.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 54228992 bytes 2024-09-14T07:13:43.953 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:43.953+0000 7fea5efdb640 -1 reset not still connected to 0x558446302a90 2024-09-14T07:13:43.954 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:43.953+0000 7fea5efdb640 -1 reset not still connected to 0x55845f031110 2024-09-14T07:13:44.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 63429632 bytes 2024-09-14T07:13:46.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 92582912 bytes 2024-09-14T07:13:47.228 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x55843f72dc70 2024-09-14T07:13:47.228 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x55844078e680 2024-09-14T07:13:47.229 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x558440d93860 2024-09-14T07:13:47.229 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:13:47.229 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:13:47.229 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:13:47.229 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:47.229+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:13:50.452 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:50.453+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:13:50.457 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:50.457+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b6a90 2024-09-14T07:13:50.463 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:13:50.465+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b7ee0 2024-09-14T07:13:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:13:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:13:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:13:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:13:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_30 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 journal 128M 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_30 128M 2024-09-14T07:13:50.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:13:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_30 2024-09-14T07:13:51.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:13:51.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:51.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_30 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:13:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:13:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=13 2024-09-14T07:13:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:13:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:13:53.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 37259264 bytes 2024-09-14T07:13:54.536 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:54.537+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03930 2024-09-14T07:13:54.537 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:54.537+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:13:54.537 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:13:54.537+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:13:57.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 75907072 bytes 2024-09-14T07:14:00.613 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.605+0000 7f99ede58640 -1 reset not still connected to 0x559211560820 2024-09-14T07:14:00.613 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.605+0000 7f99ede58640 -1 reset not still connected to 0x5592115ecea0 2024-09-14T07:14:00.618 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.621+0000 7f99ede58640 -1 reset not still connected to 0x559216788750 2024-09-14T07:14:00.619 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.621+0000 7f99ede58640 -1 reset not still connected to 0x5592168c52b0 2024-09-14T07:14:00.619 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.621+0000 7f99ede58640 -1 reset not still connected to 0x5592168c5790 2024-09-14T07:14:00.619 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.621+0000 7f99ede58640 -1 reset not still connected to 0x55922205cdd0 2024-09-14T07:14:00.619 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.621+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b6a90 2024-09-14T07:14:00.619 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:00.621+0000 7f99ede58640 -1 reset not still connected to 0x55923a4b7ee0 2024-09-14T07:14:00.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 139548672 bytes 2024-09-14T07:14:04.177 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:04.176+0000 7f99ede58640 -1 reset not still connected to 0x5592168c55f0 2024-09-14T07:14:04.180 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:04.180+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:14:04.180 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:04.180+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:14:04.180 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:04.180+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:14:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:14:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_31 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 journal 128M 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:14:04.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_31 128M 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_31 2024-09-14T07:14:04.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:14:04.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:04.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:14:04.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:14:04.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:14:04.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:14:04.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_31 2024-09-14T07:14:04.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:04.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:04.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:14:04.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:14:04.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=13 2024-09-14T07:14:04.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:14:04.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:14:04.614 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:14:04.612+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:14:06.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:06.292+0000 7fea5efdb640 -1 reset not still connected to 0x55843f72dc70 2024-09-14T07:14:06.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:06.296+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:14:06.295 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:06.296+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:14:06.295 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:06.296+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:14:06.295 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:06.296+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:14:06.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 37953536 bytes 2024-09-14T07:14:09.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 71751680 bytes 2024-09-14T07:14:10.229 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:10.228+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca4b5110 2024-09-14T07:14:10.229 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:10.228+0000 7f0df4fca640 -1 reset not still connected to 0x5615cbb055f0 2024-09-14T07:14:10.230 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:10.232+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf1d92b0 2024-09-14T07:14:12.503 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:14:12.501+0000 7f7dd9c68640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 7 2024-09-14T07:14:12.504 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:14:12.505+0000 7f7dd9467640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 6 2024-09-14T07:14:12.766 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:14:12.766 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:14:12.766 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T07:14:12.766 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:14:12.767 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:14:12.767 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:14:12.765+0000 7f32fccf0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 86860) UID: 1000 2024-09-14T07:14:12.871 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:14:12.869+0000 7f32f54e1640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x559170ff8e00 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:14:12.871 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:14:12.869+0000 7f32f54e1640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x559170ff8e00 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:14:12.871 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:14:12.869+0000 7f32f54e1640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5591751c6000 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:14:12.871 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:14:12.869+0000 7f32f54e1640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5591751c6000 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:14:14.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 157084672 bytes 2024-09-14T07:14:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 184143872 bytes 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_32 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 journal 128M 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_32 128M 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:14:17.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:14:17.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:14:17.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:14:17.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:17.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_32 2024-09-14T07:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:14:18.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:14:18.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_32 2024-09-14T07:14:18.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:18.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:18.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:14:18.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:14:18.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:14:18.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:14:18.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:14:18.869 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:14:18.869 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:14:18.869 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:14:18.871 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:14:18.872 INFO:tasks.rbd_mirror_thrash:waiting for 89 secs before thrashing 2024-09-14T07:14:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 69326848 bytes 2024-09-14T07:14:25.507 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:25.504+0000 7fea5efdb640 -1 reset not still connected to 0x558440d93860 2024-09-14T07:14:25.508 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:25.504+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:14:25.508 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:25.504+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:14:25.508 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:25.504+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:14:25.508 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:25.504+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:14:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:14:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:14:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_33 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 journal 128M 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_33 128M 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_33 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:14:26.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_33 2024-09-14T07:14:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:26.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:14:26.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:14:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:14:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:14:26.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:14:27.339 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:14:27.337+0000 7fb3e614e640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:14:29.145 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:29.144+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca52fd40 2024-09-14T07:14:29.145 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:29.144+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685e10 2024-09-14T07:14:29.147 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:29.148+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf164ea0 2024-09-14T07:14:29.147 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:29.148+0000 7f0df4fca640 -1 reset not still connected to 0x5615f2894dd0 2024-09-14T07:14:31.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 58421248 bytes 2024-09-14T07:14:34.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:14:34.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:14:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:14:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:14:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:14:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_34 2024-09-14T07:14:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 journal 128M 2024-09-14T07:14:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:14:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:14:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:14:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_34 128M 2024-09-14T07:14:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:14:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:14:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:14:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:14:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_34 2024-09-14T07:14:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:14:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:14:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:14:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:14:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:14:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_34 2024-09-14T07:14:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:14:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:14:34.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:14:34.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:14:34.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:14:37.377 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.375+0000 7fea5efdb640 -1 reset not still connected to 0x55843f0dbba0 2024-09-14T07:14:37.377 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.375+0000 7fea5efdb640 -1 reset not still connected to 0x55843f256680 2024-09-14T07:14:37.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.375+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69ca90 2024-09-14T07:14:37.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.375+0000 7fea5efdb640 -1 reset not still connected to 0x55843f72dc70 2024-09-14T07:14:37.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.375+0000 7fea5efdb640 -1 reset not still connected to 0x55844078e680 2024-09-14T07:14:37.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.379+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:14:37.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.379+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:14:37.381 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.379+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:14:37.381 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:37.379+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:14:40.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 78924800 bytes 2024-09-14T07:14:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 90518528 bytes 2024-09-14T07:14:43.291 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x55843d6f2680 2024-09-14T07:14:43.291 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69de10 2024-09-14T07:14:43.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x558440da8410 2024-09-14T07:14:43.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03e10 2024-09-14T07:14:43.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7940d0 2024-09-14T07:14:43.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:14:43.294 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:43.291+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a35f0 2024-09-14T07:14:45.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 125299712 bytes 2024-09-14T07:14:46.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 144600064 bytes 2024-09-14T07:14:46.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:14:46.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:14:46.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:14:46.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:14:46.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_35 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 journal 128M 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_35 128M 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_35 2024-09-14T07:14:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_35 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:14:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:14:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:14:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:14:46.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:14:51.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 73747456 bytes 2024-09-14T07:14:54.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 99502080 bytes 2024-09-14T07:14:55.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_36 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 journal 128M 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_36 128M 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:14:55.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:14:55.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:14:55.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:14:55.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_36 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_36 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:14:55.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:14:55.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:14:55.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:14:55.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:14:55.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:14:56.087 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:56.083+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:14:56.088 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:56.083+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:14:57.144 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:57.143+0000 7f0df4fca640 -1 reset not still connected to 0x5615cbb04410 2024-09-14T07:14:57.144 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:57.143+0000 7f0df4fca640 -1 reset not still connected to 0x5615cbb05ee0 2024-09-14T07:14:57.146 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:57.143+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685110 2024-09-14T07:14:57.147 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:14:57.147+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf1d8ea0 2024-09-14T07:14:57.446 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:14:57.440+0000 7f7c359f7640 -1 reset not still connected to 0x55ba945b71e0 2024-09-14T07:14:57.446 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:14:57.440+0000 7f7c359f7640 -1 reset not still connected to 0x55ba9602b2b0 2024-09-14T07:14:57.446 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:14:57.440+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616f1e0 2024-09-14T07:14:57.446 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:14:57.440+0000 7f7c359f7640 -1 reset not still connected to 0x55babe999380 2024-09-14T07:14:57.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 19678208 bytes 2024-09-14T07:14:58.887 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:58.883+0000 7fea5efdb640 -1 reset not still connected to 0x558440d98340 2024-09-14T07:14:58.888 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:58.887+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:14:58.888 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:14:58.887+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:15:00.279 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:15:00.280+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341ed80 finish: resending after timeout 2024-09-14T07:15:00.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 52598784 bytes 2024-09-14T07:15:01.784 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:15:01.784+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca55a481a0 2024-09-14T07:15:01.784 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:15:01.784+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca5fbac680 2024-09-14T07:15:03.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 70466560 bytes 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_37 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 journal 128M 2024-09-14T07:15:04.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_37 128M 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:05.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:05.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_37 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:15:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_37 2024-09-14T07:15:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:15:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:15:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:15:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:15:05.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:15:05.280 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:15:05.280+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341ed80 finish: resending after timeout 2024-09-14T07:15:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 587776 bytes 2024-09-14T07:15:05.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 1206272 bytes 2024-09-14T07:15:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 18198528 bytes 2024-09-14T07:15:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 21760000 bytes 2024-09-14T07:15:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 36793344 bytes 2024-09-14T07:15:11.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:600 IOs, 73299968 bytes 2024-09-14T07:15:12.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:15:12.212+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341ed80 finish: resending after timeout 2024-09-14T07:15:12.686 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x55843d6f2680 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69de10 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x558440da8410 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03e10 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x558446184410 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7940d0 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:15:12.687 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a35f0 2024-09-14T07:15:12.688 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:12.686+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10000 2024-09-14T07:15:14.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:700 IOs, 106999808 bytes 2024-09-14T07:15:14.692 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:14.686+0000 7f99ede58640 -1 reset not still connected to 0x5592118cf040 2024-09-14T07:15:14.692 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:14.690+0000 7f99ede58640 -1 reset not still connected to 0x5592168c55f0 2024-09-14T07:15:14.692 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:14.690+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:15:14.693 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:14.690+0000 7f99ede58640 -1 reset not still connected to 0x559238821e10 2024-09-14T07:15:14.693 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:14.690+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:15:14.695 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:14.694+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:15:15.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_38 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 journal 128M 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:15:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_38 128M 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:15:15.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:15:15.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:15.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_38 2024-09-14T07:15:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:15:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:15:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:15:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:15:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:15:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_38 2024-09-14T07:15:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:15:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:15:15.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=11 2024-09-14T07:15:15.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:15:15.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:15:15.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.378+0000 7fea5efdb640 -1 reset not still connected to 0x55844076a8f0 2024-09-14T07:15:15.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.378+0000 7fea5efdb640 -1 reset not still connected to 0x558440d935f0 2024-09-14T07:15:15.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.378+0000 7fea5efdb640 -1 reset not still connected to 0x558446185ba0 2024-09-14T07:15:15.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.378+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7955f0 2024-09-14T07:15:15.380 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.378+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:15:15.682 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.678+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7940d0 2024-09-14T07:15:15.682 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.678+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:15:15.682 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.678+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a35f0 2024-09-14T07:15:15.682 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:15.678+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10000 2024-09-14T07:15:16.737 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:16.734+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10000 2024-09-14T07:15:17.212 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:15:17.212+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341ed80 finish: resending after timeout 2024-09-14T07:15:18.250 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.246+0000 7fea5efdb640 -1 reset not still connected to 0x55843d6f2680 2024-09-14T07:15:18.250 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.246+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69de10 2024-09-14T07:15:18.251 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.246+0000 7fea5efdb640 -1 reset not still connected to 0x558440da8410 2024-09-14T07:15:18.251 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.246+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03e10 2024-09-14T07:15:18.252 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.250+0000 7fea5efdb640 -1 reset not still connected to 0x558446184410 2024-09-14T07:15:18.252 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.250+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7940d0 2024-09-14T07:15:18.252 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.250+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:15:18.252 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.250+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a35f0 2024-09-14T07:15:18.252 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.250+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10000 2024-09-14T07:15:18.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 45544448 bytes 2024-09-14T07:15:18.517 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:18.514+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:15:21.473 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:21.466+0000 7f99ede58640 -1 reset not still connected to 0x559211151860 2024-09-14T07:15:21.474 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:21.470+0000 7f99ede58640 -1 reset not still connected to 0x55921197e750 2024-09-14T07:15:21.477 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:21.474+0000 7f99ede58640 -1 reset not still connected to 0x559218b04ea0 2024-09-14T07:15:21.477 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:21.474+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2680 2024-09-14T07:15:22.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 75810816 bytes 2024-09-14T07:15:22.448 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:22.446+0000 7f99ede58640 -1 reset not still connected to 0x559211151860 2024-09-14T07:15:22.448 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:22.446+0000 7f99ede58640 -1 reset not still connected to 0x55921197e750 2024-09-14T07:15:22.450 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:22.446+0000 7f99ede58640 -1 reset not still connected to 0x559218b04ea0 2024-09-14T07:15:22.451 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:22.446+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2680 2024-09-14T07:15:22.456 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:15:22.460+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b341ed80 finish: resending after timeout 2024-09-14T07:15:23.323 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:23.318+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:15:23.324 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:23.318+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:15:23.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8581754683263677) > weight (0.25) 2024-09-14T07:15:23.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9474458998672198) > weight (0.25) 2024-09-14T07:15:23.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7643489774277865) > weight (0.25) 2024-09-14T07:15:23.979 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.44840787684676564) > weight (0.25) 2024-09-14T07:15:23.979 INFO:tasks.rbd_mirror_thrash:waiting for 103 secs before thrashing 2024-09-14T07:15:25.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 114169856 bytes 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_39 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 journal 128M 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_39 128M 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:15:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:15:26.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:26.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_39 2024-09-14T07:15:26.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_39 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:26.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:26.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:15:26.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:15:26.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=11 2024-09-14T07:15:26.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:15:26.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:15:26.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 2648064 bytes 2024-09-14T07:15:27.982 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:27.978+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:15:27.982 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:27.978+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:15:27.982 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:27.978+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7948f0 2024-09-14T07:15:27.982 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:27.978+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a2000 2024-09-14T07:15:27.982 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:27.978+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10ea0 2024-09-14T07:15:27.982 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:27.978+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:15:30.831 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:30.826+0000 7f0df4fca640 -1 reset not still connected to 0x561600c38820 2024-09-14T07:15:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 76577792 bytes 2024-09-14T07:15:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 81863680 bytes 2024-09-14T07:15:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:15:33.547+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aac540 finish: resending after timeout 2024-09-14T07:15:35.157 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.158+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:15:35.157 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.158+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:15:35.157 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.158+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32d00 2024-09-14T07:15:35.893 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.894+0000 7fea5efdb640 -1 reset not still connected to 0x5584404e7520 2024-09-14T07:15:35.894 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.894+0000 7fea5efdb640 -1 reset not still connected to 0x558440d981a0 2024-09-14T07:15:35.894 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.894+0000 7fea5efdb640 -1 reset not still connected to 0x558440d98340 2024-09-14T07:15:35.894 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.894+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:15:35.899 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.902+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:15:35.900 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:35.902+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32d00 2024-09-14T07:15:36.011 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:36.014+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7955f0 2024-09-14T07:15:36.013 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:36.014+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff4b5f0 2024-09-14T07:15:36.021 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:36.022+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:15:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 139069440 bytes 2024-09-14T07:15:36.555 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:36.558+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:15:36.555 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:36.558+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a35f0 2024-09-14T07:15:36.556 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:36.558+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10000 2024-09-14T07:15:37.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:15:37.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:15:37.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:15:37.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:15:37.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_40 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 journal 128M 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_40 128M 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:37.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:37.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_40 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_40 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:15:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:15:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=10 2024-09-14T07:15:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:15:37.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:15:38.894 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.894+0000 7fea5efdb640 -1 reset not still connected to 0x55843d6f2680 2024-09-14T07:15:38.895 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.894+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69de10 2024-09-14T07:15:38.895 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.894+0000 7fea5efdb640 -1 reset not still connected to 0x558440da8410 2024-09-14T07:15:38.895 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.894+0000 7fea5efdb640 -1 reset not still connected to 0x558440e03e10 2024-09-14T07:15:38.895 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.894+0000 7fea5efdb640 -1 reset not still connected to 0x558446184410 2024-09-14T07:15:38.899 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.902+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7940d0 2024-09-14T07:15:38.899 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.902+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cd520 2024-09-14T07:15:38.900 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.902+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a35f0 2024-09-14T07:15:38.900 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:15:38.902+0000 7fea5efdb640 -1 reset not still connected to 0x558470a10000 2024-09-14T07:15:39.075 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:39.070+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:15:39.076 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:15:39.074+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:15:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 68368384 bytes 2024-09-14T07:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:15:47.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_41 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 journal 128M 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_41 128M 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:15:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_41 2024-09-14T07:15:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:15:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_41 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:15:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:15:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=13 2024-09-14T07:15:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:15:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:15:47.873 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:15:47.873 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:15:47.874 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T07:15:47.874 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:15:47.875+0000 7f2050683640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 89393) UID: 1000 2024-09-14T07:15:47.937 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:15:47.939+0000 7f2048e74640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x557622bbd180 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:15:47.937 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:15:47.939+0000 7f2048e74640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x557622bbd180 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:15:48.874 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:15:48.875 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:15:48.920 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:48.921+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca52fd40 2024-09-14T07:15:48.922 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:48.921+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685e10 2024-09-14T07:15:48.923 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:48.921+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf164ea0 2024-09-14T07:15:48.923 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:48.921+0000 7f0df4fca640 -1 reset not still connected to 0x5615f2894dd0 2024-09-14T07:15:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 51848192 bytes 2024-09-14T07:15:54.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 113483776 bytes 2024-09-14T07:15:54.976 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:15:54.976 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:15:54.976 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:15:54.979 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:15:54.979 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing 2024-09-14T07:15:57.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 148712448 bytes 2024-09-14T07:15:59.893 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:59.893+0000 7f0df4fca640 -1 reset not still connected to 0x5615ca52fd40 2024-09-14T07:15:59.894 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:59.893+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685e10 2024-09-14T07:15:59.894 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:59.893+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf164ea0 2024-09-14T07:15:59.894 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:59.893+0000 7f0df4fca640 -1 reset not still connected to 0x5615d9eae1a0 2024-09-14T07:15:59.894 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:15:59.893+0000 7f0df4fca640 -1 reset not still connected to 0x5615f2894dd0 2024-09-14T07:16:00.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:00.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_42 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 journal 128M 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_42 128M 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_42 2024-09-14T07:16:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_42 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:16:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:00.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=6 2024-09-14T07:16:00.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:00.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:02.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 23364608 bytes 2024-09-14T07:16:02.662 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:02.661+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7955f0 2024-09-14T07:16:02.662 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:02.661+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff4b5f0 2024-09-14T07:16:02.662 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:02.661+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:16:03.058 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:03.057+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:16:04.685 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:04.685+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a2000 2024-09-14T07:16:04.685 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:04.685+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_43 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 journal 128M 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:06.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_43 128M 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_43 2024-09-14T07:16:06.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:06.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_43 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:16:06.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:06.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:16:06.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:06.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:06.999 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:06.999+0000 7f7c359f7640 -1 reset not still connected to 0x55ba92bd80d0 2024-09-14T07:16:06.999 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:06.999+0000 7f7c359f7640 -1 reset not still connected to 0x55ba945b71e0 2024-09-14T07:16:07.003 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:07.003+0000 7f7c359f7640 -1 reset not still connected to 0x55ba9602b2b0 2024-09-14T07:16:07.003 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:07.003+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616f1e0 2024-09-14T07:16:10.487 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:10.486+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aad2c0 finish: resending after timeout 2024-09-14T07:16:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 42308608 bytes 2024-09-14T07:16:12.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 58868736 bytes 2024-09-14T07:16:12.418 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:12.418+0000 7f7c245b4640 -1 bdev(0x55ba904d4c00 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 2 2024-09-14T07:16:13.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 81525760 bytes 2024-09-14T07:16:15.487 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:15.486+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aad2c0 finish: resending after timeout 2024-09-14T07:16:17.460 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:17.458+0000 7f7c225b0640 -1 bdev(0x55ba904d4c00 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 4 2024-09-14T07:16:17.462 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:16:17.462+0000 7f7c24db5640 -1 bdev(0x55ba904d4c00 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 3 2024-09-14T07:16:18.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:18.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:18.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_44 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 journal 128M 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_44 128M 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_44 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_44 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:18.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:16:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:16:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:19.042 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.041+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:16:19.042 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.041+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:16:19.042 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.041+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:16:19.042 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.041+0000 7f99ede58640 -1 reset not still connected to 0x55921673ab60 2024-09-14T07:16:19.044 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.045+0000 7f99ede58640 -1 reset not still connected to 0x559216789110 2024-09-14T07:16:19.045 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.045+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:16:19.045 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:19.045+0000 7f99ede58640 -1 reset not still connected to 0x55923a441c70 2024-09-14T07:16:19.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 10024960 bytes 2024-09-14T07:16:22.478 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:22.478+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aad2c0 finish: resending after timeout 2024-09-14T07:16:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 68845568 bytes 2024-09-14T07:16:26.994 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.992+0000 7fea5efdb640 -1 reset not still connected to 0x55844076a8f0 2024-09-14T07:16:26.995 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.992+0000 7fea5efdb640 -1 reset not still connected to 0x558440d935f0 2024-09-14T07:16:27.069 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.996+0000 7fea5efdb640 -1 reset not still connected to 0x558446185ba0 2024-09-14T07:16:27.069 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.996+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7955f0 2024-09-14T07:16:27.069 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.996+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff4b5f0 2024-09-14T07:16:27.070 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.996+0000 7fea5efdb640 -1 reset not still connected to 0x558471515860 2024-09-14T07:16:27.070 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.996+0000 7fea5efdb640 -1 reset not still connected to 0x558471f25860 2024-09-14T07:16:27.070 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:26.996+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:16:27.280 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.276+0000 7fea5efdb640 -1 reset not still connected to 0x55844078ef70 2024-09-14T07:16:27.280 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.276+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50d00 2024-09-14T07:16:27.283 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.284+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:16:27.283 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.284+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cc680 2024-09-14T07:16:27.283 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.284+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:16:27.283 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.284+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:16:27.285 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:27.284+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:16:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 101645312 bytes 2024-09-14T07:16:27.839 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.836+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:16:27.840 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.836+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:16:27.840 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.836+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:16:27.840 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.840+0000 7f99ede58640 -1 reset not still connected to 0x559216789110 2024-09-14T07:16:27.844 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.840+0000 7f99ede58640 -1 reset not still connected to 0x55923a441c70 2024-09-14T07:16:27.890 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x5592115e5c70 2024-09-14T07:16:27.890 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:16:27.890 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:16:27.891 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:16:27.891 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x55921673ab60 2024-09-14T07:16:27.891 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x559216789110 2024-09-14T07:16:27.891 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:16:27.891 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:27.888+0000 7f99ede58640 -1 reset not still connected to 0x55923a441c70 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_45 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 journal 128M 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:27.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_45 128M 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:16:27.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_45 2024-09-14T07:16:28.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:28.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_45 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:16:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:28.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T07:16:28.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:28.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:28.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 3266560 bytes 2024-09-14T07:16:29.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 16039936 bytes 2024-09-14T07:16:31.868 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:31.870+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aad2c0 finish: resending after timeout 2024-09-14T07:16:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 61686784 bytes 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_46 2024-09-14T07:16:33.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 journal 128M 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_46 128M 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_46 2024-09-14T07:16:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_46 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=7 2024-09-14T07:16:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:36.343 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.340+0000 7fea5efdb640 -1 reset not still connected to 0x55844078ef70 2024-09-14T07:16:36.343 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.340+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50d00 2024-09-14T07:16:36.346 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.344+0000 7fea5efdb640 -1 reset not still connected to 0x558446303860 2024-09-14T07:16:36.346 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.344+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:16:36.346 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.344+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cc680 2024-09-14T07:16:36.346 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.344+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:16:36.351 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.348+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:16:36.351 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:36.348+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:16:36.869 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:36.870+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aad2c0 finish: resending after timeout 2024-09-14T07:16:38.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 80386048 bytes 2024-09-14T07:16:38.493 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:38.494+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b1c72f00 finish: resending after timeout 2024-09-14T07:16:38.625 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.624+0000 7fea5efdb640 -1 reset not still connected to 0x55844078ef70 2024-09-14T07:16:38.626 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.624+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50d00 2024-09-14T07:16:38.627 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.628+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:16:38.627 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.628+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cc680 2024-09-14T07:16:38.627 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.628+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:16:38.627 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.628+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:16:38.627 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:38.628+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_47 2024-09-14T07:16:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 journal 128M 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_47 128M 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:40.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_47 2024-09-14T07:16:40.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:40.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:40.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:40.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_47 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:16:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:40.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=11 2024-09-14T07:16:40.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:40.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:40.902 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:40.902+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b8b9ee40 finish: resending after timeout 2024-09-14T07:16:43.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 41295872 bytes 2024-09-14T07:16:45.902 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:45.902+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b8b9ee40 finish: resending after timeout 2024-09-14T07:16:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 67374080 bytes 2024-09-14T07:16:48.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 93389824 bytes 2024-09-14T07:16:49.142 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.140+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:16:49.427 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55843f0db6c0 2024-09-14T07:16:49.427 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55843f0db860 2024-09-14T07:16:49.427 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55843f0dbba0 2024-09-14T07:16:49.427 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69c340 2024-09-14T07:16:49.427 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69ca90 2024-09-14T07:16:49.427 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55843f72dc70 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x558440d93860 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x558445208750 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55844520aa90 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7948f0 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a2000 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a2d00 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a31e0 2024-09-14T07:16:49.428 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:49.424+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:16:50.565 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:16:50.566+0000 7fb3e414a640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:16:50.569 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:16:50.570+0000 7fb3e9154640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:16:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:16:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_48 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 journal 128M 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_48 128M 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:16:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_48 2024-09-14T07:16:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_48 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:16:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:16:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:16:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:16:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:16:53.878 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x5592118cef70 2024-09-14T07:16:53.878 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x5592118cf040 2024-09-14T07:16:53.878 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x559214ba8750 2024-09-14T07:16:53.878 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x559214ba92b0 2024-09-14T07:16:53.878 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x5592168c55f0 2024-09-14T07:16:53.879 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x5592179252b0 2024-09-14T07:16:53.879 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x559217d4bd40 2024-09-14T07:16:53.879 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.876+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:16:53.881 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.880+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2a90 2024-09-14T07:16:53.881 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.880+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2d00 2024-09-14T07:16:53.882 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.880+0000 7f99ede58640 -1 reset not still connected to 0x559238821e10 2024-09-14T07:16:53.882 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.880+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:16:53.882 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.880+0000 7f99ede58640 -1 reset not still connected to 0x559240c86820 2024-09-14T07:16:53.882 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:53.880+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:16:54.069 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.068+0000 7fea5efdb640 -1 reset not still connected to 0x55844078ef70 2024-09-14T07:16:54.072 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.068+0000 7fea5efdb640 -1 reset not still connected to 0x5584448ccd00 2024-09-14T07:16:54.072 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.068+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50d00 2024-09-14T07:16:54.073 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.068+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:16:54.073 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.068+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:16:54.073 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.072+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:16:54.073 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:54.072+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:16:54.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 32770048 bytes 2024-09-14T07:16:55.326 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:16:55.326+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b1c72f00 finish: resending after timeout 2024-09-14T07:16:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 45414400 bytes 2024-09-14T07:16:57.499 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.496+0000 7f99ede58640 -1 reset not still connected to 0x559210517110 2024-09-14T07:16:57.502 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.500+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:16:57.502 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.500+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:16:57.502 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.500+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:16:57.502 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.500+0000 7f99ede58640 -1 reset not still connected to 0x55921673ab60 2024-09-14T07:16:57.503 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.504+0000 7f99ede58640 -1 reset not still connected to 0x559216789110 2024-09-14T07:16:57.505 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:16:57.504+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:16:57.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 98194432 bytes 2024-09-14T07:16:59.056 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:59.056+0000 7fea5efdb640 -1 reset not still connected to 0x558440d98340 2024-09-14T07:16:59.056 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:59.056+0000 7fea5efdb640 -1 reset not still connected to 0x558446302340 2024-09-14T07:16:59.057 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:59.056+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7941a0 2024-09-14T07:16:59.057 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:59.056+0000 7fea5efdb640 -1 reset not still connected to 0x558461aaaf70 2024-09-14T07:16:59.057 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:59.056+0000 7fea5efdb640 -1 reset not still connected to 0x55847134f860 2024-09-14T07:16:59.057 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:16:59.056+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32d00 2024-09-14T07:17:00.257 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:00.256+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:17:00.258 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:00.256+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cc680 2024-09-14T07:17:00.258 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:00.256+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:17:00.258 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:00.256+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:17:00.258 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:00.256+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:17:00.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 143083520 bytes 2024-09-14T07:17:00.327 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:00.325+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b1c72f00 finish: resending after timeout 2024-09-14T07:17:00.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 148551680 bytes 2024-09-14T07:17:02.549 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.544+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:17:02.549 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.548+0000 7f99ede58640 -1 reset not still connected to 0x559216789110 2024-09-14T07:17:02.549 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.548+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:17:02.730 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.728+0000 7f99ede58640 -1 reset not still connected to 0x5592118cf450 2024-09-14T07:17:02.731 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.732+0000 7f99ede58640 -1 reset not still connected to 0x559214ba9930 2024-09-14T07:17:02.731 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.732+0000 7f99ede58640 -1 reset not still connected to 0x559217d4a1a0 2024-09-14T07:17:02.731 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.732+0000 7f99ede58640 -1 reset not still connected to 0x55922dbc95f0 2024-09-14T07:17:02.731 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:02.732+0000 7f99ede58640 -1 reset not still connected to 0x55923a440dd0 2024-09-14T07:17:02.836 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:02.832+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc685e10 2024-09-14T07:17:02.836 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:02.832+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf164ea0 2024-09-14T07:17:02.836 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:02.832+0000 7f0df4fca640 -1 reset not still connected to 0x5615d2436000 2024-09-14T07:17:02.836 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:02.832+0000 7f0df4fca640 -1 reset not still connected to 0x5615d9eae1a0 2024-09-14T07:17:02.836 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:02.832+0000 7f0df4fca640 -1 reset not still connected to 0x5615f2894d00 2024-09-14T07:17:02.837 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:02.832+0000 7f0df4fca640 -1 reset not still connected to 0x5615f2894dd0 2024-09-14T07:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:17:03.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_49 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 journal 128M 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_49 128M 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:03.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_49 2024-09-14T07:17:03.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:17:03.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:03.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_49 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:17:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:17:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=6 2024-09-14T07:17:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:17:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:17:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 30279680 bytes 2024-09-14T07:17:06.980 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7029711149057464) > weight (0.25) 2024-09-14T07:17:06.980 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-14T07:17:06.980 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-14T07:17:06.981 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T07:17:06.981 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:17:06.981+0000 7f04d1d6c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 83873) UID: 1000 2024-09-14T07:17:07.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:07.545+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b8b9ee40 finish: resending after timeout 2024-09-14T07:17:09.201 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:17:09.201+0000 7fd596e7d640 -1 rbd::mirror::ImageReplayer: 0x55f615424500 [3/ace9bd35-7f3f-43a1-ac1f-213c64b8a38a] start: already running 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_50 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 journal 128M 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n journal ']' 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_50 128M 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 128M ']' 2024-09-14T07:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=128M 2024-09-14T07:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_50 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ is_pool_mirror_mode_image mirror 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' pool = image ']' 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ case "${pool}" in 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 1 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n '' ']' 2024-09-14T07:17:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_50 2024-09-14T07:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:17:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=9 2024-09-14T07:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:17:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:17:11.150 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:11.149+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b1c72f00 finish: resending after timeout 2024-09-14T07:17:11.204 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:11.199+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69d5f0 2024-09-14T07:17:11.204 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:11.203+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:17:11.205 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:11.203+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cc680 2024-09-14T07:17:11.205 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:11.203+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:17:11.205 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:11.203+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:17:11.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 21093376 bytes 2024-09-14T07:17:11.603 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:11.599+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:17:11.613 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:11.603+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:17:12.080 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:12.075+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:17:12.080 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:12.079+0000 7f99ede58640 -1 reset not still connected to 0x559216789110 2024-09-14T07:17:12.297 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:12.295+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:17:12.297 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:12.295+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:17:12.981 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-14T07:17:12.981 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:17:12.982 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-14T07:17:12.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-14T07:17:12.982 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-14T07:17:12.985 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-14T07:17:12.985 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing 2024-09-14T07:17:13.776 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:13.775+0000 7f0df4fca640 -1 reset not still connected to 0x5615faaaa1a0 2024-09-14T07:17:15.138 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:15.135+0000 7fea5efdb640 -1 reset not still connected to 0x558471515860 2024-09-14T07:17:15.138 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:15.135+0000 7fea5efdb640 -1 reset not still connected to 0x558471f25860 2024-09-14T07:17:15.138 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:15.135+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:17:15.138 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:15.135+0000 7fea5efdb640 -1 reset not still connected to 0x558475c5c410 2024-09-14T07:17:16.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 68955136 bytes 2024-09-14T07:17:16.516 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:16.515+0000 7f99ede58640 -1 reset not still connected to 0x559211151860 2024-09-14T07:17:16.516 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:16.515+0000 7f99ede58640 -1 reset not still connected to 0x55921197e750 2024-09-14T07:17:16.516 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:16.515+0000 7f99ede58640 -1 reset not still connected to 0x559216788dd0 2024-09-14T07:17:16.518 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:16.515+0000 7f99ede58640 -1 reset not still connected to 0x559218b04ea0 2024-09-14T07:17:16.518 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:16.515+0000 7f99ede58640 -1 reset not still connected to 0x55922205c410 2024-09-14T07:17:16.518 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:16.515+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2680 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_pool_images cluster2 mirror 50 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_count=50 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local count 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_count=0 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:17:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 40 2024-09-14T07:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 1 -ne 1 2024-09-14T07:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-14T07:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'images: ' 2024-09-14T07:17:18.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 2 2024-09-14T07:17:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:17:18.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ count=50 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 50 = 50 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_pool_healthy cluster2 mirror 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state 2024-09-14T07:17:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 40 2024-09-14T07:17:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:17:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 1 -ne 1 2024-09-14T07:17:18.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-14T07:17:18.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'image health:' 2024-09-14T07:17:18.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 3 2024-09-14T07:17:18.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:17:18.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=WARNING 2024-09-14T07:17:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test WARNING = ERROR 2024-09-14T07:17:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test WARNING = OK 2024-09-14T07:17:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:17:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 -ne 1 2024-09-14T07:17:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 30 2024-09-14T07:17:19.984 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x55844078ef70 2024-09-14T07:17:19.984 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x5584448ccd00 2024-09-14T07:17:19.984 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50d00 2024-09-14T07:17:19.985 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x558461aab520 2024-09-14T07:17:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x5584646cc680 2024-09-14T07:17:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:17:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:17:19.986 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:19.983+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:17:21.364 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:21.361+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b8b9ee40 finish: resending after timeout 2024-09-14T07:17:21.370 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:21.369+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b391a3c0 finish: resending after timeout 2024-09-14T07:17:24.377 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x55843f69c0d0 2024-09-14T07:17:24.377 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x55844076a8f0 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x55844078e4e0 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x558440d935f0 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x558446185ba0 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7955f0 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff4b5f0 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x558471515860 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x558471f25860 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x558472e32270 2024-09-14T07:17:24.378 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:24.375+0000 7fea5efdb640 -1 reset not still connected to 0x558475c5c410 2024-09-14T07:17:25.259 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:25.255+0000 7fea5efdb640 -1 reset not still connected to 0x55846ff9d450 2024-09-14T07:17:25.259 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:25.255+0000 7fea5efdb640 -1 reset not still connected to 0x558472e321a0 2024-09-14T07:17:25.259 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:25.255+0000 7fea5efdb640 -1 reset not still connected to 0x558472e33ad0 2024-09-14T07:17:26.307 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:26.303+0000 7f0df4fca640 -1 reset not still connected to 0x5615cbb05450 2024-09-14T07:17:26.307 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:26.303+0000 7f0df4fca640 -1 reset not still connected to 0x5615cc684c30 2024-09-14T07:17:26.308 INFO:tasks.ceph.cluster1.osd.1.smithi129.stderr:2024-09-14T07:17:26.303+0000 7f0df4fca640 -1 reset not still connected to 0x5615cf434f70 2024-09-14T07:17:27.210 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:27.207+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:17:27.210 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:27.207+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:17:27.210 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:27.207+0000 7f99ede58640 -1 reset not still connected to 0x559216728270 2024-09-14T07:17:27.210 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:27.207+0000 7f99ede58640 -1 reset not still connected to 0x55921673ab60 2024-09-14T07:17:27.213 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:27.211+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:17:27.216 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:27.211+0000 7f99ede58640 -1 reset not still connected to 0x55923a441c70 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x558440d995f0 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x558445208750 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x55844520aa90 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7948f0 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a2000 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a2d00 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a31e0 2024-09-14T07:17:33.771 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:17:33.767+0000 7fea5efdb640 -1 reset not still connected to 0x558472e325b0 2024-09-14T07:17:34.487 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:34.485+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b391a3c0 finish: resending after timeout 2024-09-14T07:17:41.175 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:41.173+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b391a3c0 finish: resending after timeout 2024-09-14T07:17:41.980 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.509015273292508) > weight (0.25) 2024-09-14T07:17:41.980 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4168975732726974) > weight (0.25) 2024-09-14T07:17:41.980 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T07:17:41.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T07:17:41.981 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T07:17:41.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:17:41.981+0000 7ff171e4a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 84413) UID: 1000 2024-09-14T07:17:46.176 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:46.173+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b391a3c0 finish: resending after timeout 2024-09-14T07:17:46.778 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:46.775+0000 7f99ede58640 -1 reset not still connected to 0x5592168c55f0 2024-09-14T07:17:46.778 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:46.775+0000 7f99ede58640 -1 reset not still connected to 0x559218b048f0 2024-09-14T07:17:46.779 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:46.775+0000 7f99ede58640 -1 reset not still connected to 0x5592220c2a90 2024-09-14T07:17:46.779 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:46.775+0000 7f99ede58640 -1 reset not still connected to 0x559238821e10 2024-09-14T07:17:46.779 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:46.775+0000 7f99ede58640 -1 reset not still connected to 0x55923a43fc70 2024-09-14T07:17:46.780 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:17:46.779+0000 7f99ede58640 -1 reset not still connected to 0x5592421a5520 2024-09-14T07:17:47.982 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T07:17:47.982 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:17:47.982 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T07:17:47.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T07:17:47.982 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T07:17:47.985 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T07:17:47.985 INFO:tasks.rbd_mirror_thrash:waiting for 94 secs before thrashing 2024-09-14T07:17:48.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-14T07:17:48.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'image health:' 2024-09-14T07:17:48.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 3 2024-09-14T07:17:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:17:48.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=WARNING 2024-09-14T07:17:48.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test WARNING = ERROR 2024-09-14T07:17:48.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test WARNING = OK 2024-09-14T07:17:48.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:17:48.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 -ne 1 2024-09-14T07:17:48.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 30 2024-09-14T07:17:50.272 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:50.268+0000 7f6a9d41c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55a2b3a98380 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:17:50.272 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:50.268+0000 7f6a9d41c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55a2b3a98380 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:17:51.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:17:51.176+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b391a3c0 finish: resending after timeout 2024-09-14T07:17:53.383 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:17:53.380+0000 7fb3f8d94640 -1 reset not still connected to 0x561493993ba0 2024-09-14T07:17:53.387 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:17:53.384+0000 7fb3f8d94640 -1 reset not still connected to 0x5614a2f0bee0 2024-09-14T07:17:53.387 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:17:53.384+0000 7fb3f8d94640 -1 reset not still connected to 0x5614b5844d00 2024-09-14T07:17:53.387 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:17:53.384+0000 7fb3f8d94640 -1 reset not still connected to 0x5614be3992b0 2024-09-14T07:17:53.387 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:17:53.384+0000 7fb3f8d94640 -1 reset not still connected to 0x5614bf1c4750 2024-09-14T07:17:53.387 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:17:53.384+0000 7fb3f8d94640 -1 reset not still connected to 0x5614c155f6c0 2024-09-14T07:18:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-14T07:18:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'image health:' 2024-09-14T07:18:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 3 2024-09-14T07:18:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:18:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=OK 2024-09-14T07:18:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test OK = ERROR 2024-09-14T07:18:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test OK = OK 2024-09-14T07:18:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_pool_images cluster1 mirror 50 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_count=50 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local count 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_count=0 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 40 2024-09-14T07:18:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:18:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 1 -ne 1 2024-09-14T07:18:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T07:18:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'images: ' 2024-09-14T07:18:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 2 2024-09-14T07:18:19.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ count=50 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 50 = 50 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_pool_healthy cluster1 mirror 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state 2024-09-14T07:18:19.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 40 2024-09-14T07:18:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:18:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 1 -ne 1 2024-09-14T07:18:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T07:18:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'image health:' 2024-09-14T07:18:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 3 2024-09-14T07:18:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=OK 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test OK = ERROR 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test OK = OK 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: compare many images' 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T07:18:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T07:18:19.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 07:18:19 TEST: compare many images 2024-09-14T07:18:19.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 07:18:19 TEST: compare many images 2024-09-14T07:18:19.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 50 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_1 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_1 snap 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:22.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:18:22.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:22.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:22.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:18:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2024-09-14T07:18:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:23.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:18:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:18:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:18:23.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:23.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:18:23.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:23.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-14T07:18:23.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:23.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:18:23.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-14T07:18:23.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 4 closest matches: 2024-09-14T07:18:23.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_10 2024-09-14T07:18:23.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_11 2024-09-14T07:18:23.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_13 2024-09-14T07:18:23.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_16 2024-09-14T07:18:23.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-14T07:18:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:18:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:23.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:18:24.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:24.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:24.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:24.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:24.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-14T07:18:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:24.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_19 2024-09-14T07:18:24.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:24.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:24.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:24.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:24.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:24.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-14T07:18:24.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:18:24.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:18:24.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:18:24.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:18:24.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_1 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_1 2024-09-14T07:18:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:18:24.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_1 2024-09-14T07:18:24.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:24.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_1 ']' 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_1' 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:24.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:24.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:24.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:24.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:24.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:24.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 4 closest matches: 2024-09-14T07:18:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_10 2024-09-14T07:18:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_11 2024-09-14T07:18:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_13 2024-09-14T07:18:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_16 2024-09-14T07:18:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:24.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:24.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:24.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:24.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:24.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:24.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_19 2024-09-14T07:18:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:24.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:24.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:24.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_1 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '' 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:24.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:24.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:24.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:24.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:24.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:25.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:25.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:25.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11416' '!=' '' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11416' = '' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:18:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:18:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:25.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:18:25.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:25.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:18:25.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:25.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:18:25.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:25.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:18:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:18:25.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11416 2024-09-14T07:18:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11416 2024-09-14T07:18:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:25.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:25.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:18:25.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_1 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_1 ']' 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_1' 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:25.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:25.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:25.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:25.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 4 closest matches: 2024-09-14T07:18:25.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_10 2024-09-14T07:18:25.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_11 2024-09-14T07:18:25.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_13 2024-09-14T07:18:25.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_16 2024-09-14T07:18:25.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:25.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:25.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2024-09-14T07:18:25.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:25.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:25.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:25.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:25.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:25.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:25.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:25.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:25.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:25.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_19 2024-09-14T07:18:25.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2024-09-14T07:18:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:25.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:25.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:25.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_1 2024-09-14T07:18:25.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '' 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:25.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:25.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:25.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:25.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:25.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:25.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:25.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:25.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2024-09-14T07:18:25.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:25.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:25.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:25.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:26.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:26.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:26.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:26.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:26.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:26.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:26.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:26.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:26.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11416' '!=' 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:26.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:26.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:18:26.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_1 2024-09-14T07:18:26.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:26.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:26.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:26.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_1 ']' 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_1' 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:26.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:26.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:26.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:26.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:26.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:26.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:26.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:26.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:26.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:26.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 4 closest matches: 2024-09-14T07:18:26.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_10 2024-09-14T07:18:26.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_11 2024-09-14T07:18:26.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_13 2024-09-14T07:18:26.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_16 2024-09-14T07:18:26.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:26.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:26.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:26.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:26.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_19 2024-09-14T07:18:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2024-09-14T07:18:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:26.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:26.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:26.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:26.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:26.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:26.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:26.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:26.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:26.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:27.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:27.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_1 2024-09-14T07:18:27.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:27.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '' 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:27.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:27.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:27.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:27.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:27.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2024-09-14T07:18:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:27.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11416' '!=' 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_1 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_1 ']' 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_1' 2024-09-14T07:18:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:27.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:27.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:28.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:28.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:28.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:28.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 4 closest matches: 2024-09-14T07:18:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_10 2024-09-14T07:18:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_11 2024-09-14T07:18:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_13 2024-09-14T07:18:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_16 2024-09-14T07:18:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:28.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:28.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:28.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:28.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:28.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:28.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:28.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:28.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:28.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:28.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:28.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:28.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:28.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:28.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:28.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_19 2024-09-14T07:18:28.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:28.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:28.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:28.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:28.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:28.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:28.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:28.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:28.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:28.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:28.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_1 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '' 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:28.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:28.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:28.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:28.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:28.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:28.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:28.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:28.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:28.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:28.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11416], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414], [object_number=9, tag_tid=1, entry_tid=11413]]], state=connected] 2024-09-14T07:18:28.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11416' '!=' 'object_number=8, tag_tid=1, entry_tid=11416' 2024-09-14T07:18:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_1 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_1 ']' 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_1' 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2024-09-14T07:18:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:30.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:30.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:30.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:30.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:30.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 4 closest matches: 2024-09-14T07:18:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_10 2024-09-14T07:18:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_11 2024-09-14T07:18:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_13 2024-09-14T07:18:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_16 2024-09-14T07:18:30.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:30.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:30.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:30.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:30.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:30.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:30.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_19 2024-09-14T07:18:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:30.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:30.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:30.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:30.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:30.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:30.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2024-09-14T07:18:30.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_1 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '' 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:30.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:30.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:30.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:30.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:30.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:30.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:30.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:30.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:30.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:30.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2024-09-14T07:18:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_1.status 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_1.status 2024-09-14T07:18:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:30.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:30.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2024-09-14T07:18:30.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11415], [object_number=10, tag_tid=1, entry_tid=11414]]], state=connected] 2024-09-14T07:18:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.status 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:18:30.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:18:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_1': 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b096e39da6 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b096e39da6 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:09:06 2024 2024-09-14T07:18:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:09:06 2024 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:07 2024 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b096e39da6 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 0c7d1d60-b691-4928-851a-f6ebbfbc6406 2024-09-14T07:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_1 snap 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.export 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_1.export 2024-09-14T07:18:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_1.export 2024-09-14T07:18:31.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.export 2024-09-14T07:18:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:18:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_1.export 2024-09-14T07:18:32.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:18:32.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_1.export 2024-09-14T07:18:32.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_1.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_1.export 2024-09-14T07:18:32.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:32.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:18:32.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_2 2024-09-14T07:18:32.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_2 snap 2024-09-14T07:18:32.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:32.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:32.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:32.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:32.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:35.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:18:35.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:35.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:35.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:18:36.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2024-09-14T07:18:36.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:36.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:36.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:36.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:18:36.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:18:36.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:18:36.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2024-09-14T07:18:36.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:18:36.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:18:36.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:36.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:36.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:18:36.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:18:36.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:36.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:18:36.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:36.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:18:36.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:36.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:36.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:36.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:36.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:36.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:36.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:18:36.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:18:36.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:36.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:36.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:36.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:36.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2024-09-14T07:18:36.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:36.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:18:36.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:18:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2024-09-14T07:18:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_23 2024-09-14T07:18:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_26 2024-09-14T07:18:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_28 2024-09-14T07:18:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:37.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:18:37.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:37.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:37.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:37.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:37.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2024-09-14T07:18:37.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:18:37.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:18:37.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:18:37.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:18:37.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_2 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:37.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_2 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:37.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:18:37.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_2 2024-09-14T07:18:37.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:37.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:37.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_2 ']' 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_2' 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:37.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2024-09-14T07:18:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:37.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:37.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:18:37.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2024-09-14T07:18:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:37.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:37.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:37.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:37.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:37.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:37.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:37.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:37.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_23 2024-09-14T07:18:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_26 2024-09-14T07:18:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_28 2024-09-14T07:18:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:37.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:37.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2024-09-14T07:18:37.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:37.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:37.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:37.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:37.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:37.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:37.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:37.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:37.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:37.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:37.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:37.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_2 2024-09-14T07:18:37.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '' 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:37.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:37.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:37.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:37.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:37.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:37.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:37.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:37.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:37.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:37.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:37.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:37.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:37.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:37.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:37.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:37.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:37.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:37.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:37.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:37.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10196' '!=' '' 2024-09-14T07:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10196' = '' 2024-09-14T07:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:37.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:18:37.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:37.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:18:37.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:38.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:18:38.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:18:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:18:38.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10196 2024-09-14T07:18:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 10196 2024-09-14T07:18:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_2 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_2 ']' 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_2' 2024-09-14T07:18:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:38.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:38.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:38.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:38.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:38.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:38.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:38.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:38.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:18:38.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:38.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:38.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:38.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:38.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:38.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:38.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:38.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:38.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:38.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_23 2024-09-14T07:18:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_26 2024-09-14T07:18:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_28 2024-09-14T07:18:38.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:38.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:38.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:38.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:38.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:38.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:38.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:38.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:38.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:38.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:38.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:38.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:38.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:38.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:38.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_2 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '' 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:38.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:38.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:38.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:38.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:38.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:38.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:38.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:38.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:38.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:38.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:38.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:38.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:38.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:38.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:38.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:38.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:38.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:38.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10196' '!=' 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:18:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_2 2024-09-14T07:18:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_2 ']' 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_2' 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:39.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:39.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:39.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:39.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2024-09-14T07:18:39.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:39.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:39.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:39.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:39.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:39.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:39.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:39.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:39.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:39.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:39.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:39.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:39.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:39.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:39.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:39.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:39.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:39.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:39.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:39.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:39.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:39.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:39.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:39.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:18:39.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:39.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:39.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:39.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:39.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_23 2024-09-14T07:18:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_26 2024-09-14T07:18:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_28 2024-09-14T07:18:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:39.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:39.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:39.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:39.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_2 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '' 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:39.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:39.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:39.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:39.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:39.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:39.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:39.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:39.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:39.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:39.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:39.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:39.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:39.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:39.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:39.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:39.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:39.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10196' '!=' 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_2 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_2 ']' 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_2' 2024-09-14T07:18:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:40.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:40.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:40.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2024-09-14T07:18:40.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:40.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:40.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:40.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:40.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:40.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:40.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:18:40.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:40.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:40.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:40.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:40.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_23 2024-09-14T07:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_26 2024-09-14T07:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_28 2024-09-14T07:18:40.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2024-09-14T07:18:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:40.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:40.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:40.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:40.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:40.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:40.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_2 2024-09-14T07:18:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '' 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:40.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:40.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:40.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:40.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:40.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:41.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:41.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:41.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10196' '!=' 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:18:42.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_2 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_2 ']' 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_2' 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:18:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:42.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:42.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:42.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:42.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:42.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:42.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:42.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:42.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:42.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_23 2024-09-14T07:18:42.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_26 2024-09-14T07:18:42.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_28 2024-09-14T07:18:42.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2024-09-14T07:18:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:42.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:42.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:42.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:42.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_2 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '' 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:43.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:43.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:43.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:43.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:43.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:43.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:43.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:43.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:43.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:43.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:43.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10196], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194], [object_number=9, tag_tid=1, entry_tid=10193]]], state=connected] 2024-09-14T07:18:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10196' '!=' 'object_number=8, tag_tid=1, entry_tid=10196' 2024-09-14T07:18:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:18:45.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_2 2024-09-14T07:18:45.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:45.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:45.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:45.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:45.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_2 ']' 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_2' 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:45.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:45.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:45.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:45.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:45.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:45.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:45.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:45.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:45.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:45.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:45.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:45.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:45.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:45.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:45.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:45.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-14T07:18:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:18:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2024-09-14T07:18:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:45.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:45.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:45.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:45.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:45.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:45.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:45.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 3 closest matches: 2024-09-14T07:18:45.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_23 2024-09-14T07:18:45.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_26 2024-09-14T07:18:45.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_28 2024-09-14T07:18:45.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:45.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:18:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:18:45.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:18:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:18:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:18:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:18:45.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:45.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:18:45.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2024-09-14T07:18:45.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:45.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_2 2024-09-14T07:18:45.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:45.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:45.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '' 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:45.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:45.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:45.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:45.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:45.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:45.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:45.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:45.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:45.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:45.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:45.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:45.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:18:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:45.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_2.status 2024-09-14T07:18:45.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_2.status 2024-09-14T07:18:45.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:45.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:45.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2024-09-14T07:18:45.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:18:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10195], [object_number=10, tag_tid=1, entry_tid=10194]]], state=connected] 2024-09-14T07:18:45.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.status 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:45.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:45.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:18:45.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:18:46.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_2': 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1182fc2e23ee 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1182fc2e23ee 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:18:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:09:16 2024 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:09:16 2024 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:09:16 2024 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1182fc2e23ee 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 6f654538-f738-4f37-ad20-6fa40855e451 2024-09-14T07:18:46.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_2 snap 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.export 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_2.export 2024-09-14T07:18:46.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_2.export 2024-09-14T07:18:46.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.export 2024-09-14T07:18:47.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:18:47.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_2.export 2024-09-14T07:18:47.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:18:47.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_2.export 2024-09-14T07:18:47.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_2.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_2.export 2024-09-14T07:18:47.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_3 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_3 snap 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:18:47.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:18:51.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:18:52.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2024-09-14T07:18:52.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:52.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:52.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:18:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:18:52.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:52.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:18:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2024-09-14T07:18:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:18:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:52.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:18:52.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:18:52.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:18:52.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:52.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:52.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:52.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:52.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2024-09-14T07:18:52.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:52.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status mirror/image_34 2024-09-14T07:18:52.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:52.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:52.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2024-09-14T07:18:52.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:18:52.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:18:52.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:52.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:52.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:52.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:52.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2024-09-14T07:18:52.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:18:52.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:18:52.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:18:52.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:18:52.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_3 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_3 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:52.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:18:52.986 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.44818708605351565) > weight (0.25) 2024-09-14T07:18:52.986 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.40296913962379943) > weight (0.25) 2024-09-14T07:18:52.986 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:18:52.986 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:18:52.987 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:18:52.987 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:18:52.987+0000 7f8059833640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 88284) UID: 1000 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_3 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_3 ']' 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_3' 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:53.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:53.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:53.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:53.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:53.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:53.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:53.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:53.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:53.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:53.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:53.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:53.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_34 2024-09-14T07:18:53.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2024-09-14T07:18:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:53.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:53.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:53.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:53.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:53.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:53.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:53.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:53.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:53.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:53.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_3 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '' 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:53.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:53.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:53.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:53.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:53.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:53.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:53.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:53.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:53.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:53.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2024-09-14T07:18:53.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:53.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:53.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:53.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:53.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:53.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=12537' '!=' '' 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=12537' = '' 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:18:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:18:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:18:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:18:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:18:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:18:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:18:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=12537 2024-09-14T07:18:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 12537 2024-09-14T07:18:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:18:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:18:53.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_3 2024-09-14T07:18:53.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_3 ']' 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_3' 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:53.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:53.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:53.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:53.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:53.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:53.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2024-09-14T07:18:53.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:53.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:53.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:53.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:53.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:53.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:53.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:53.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:53.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:53.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:53.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:53.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_34 2024-09-14T07:18:53.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:53.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:53.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:53.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:53.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:53.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:53.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:53.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:53.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:53.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:54.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_3 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '' 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:54.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:54.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:54.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:54.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:54.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:54.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:54.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:54.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2024-09-14T07:18:54.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:54.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:54.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:54.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:54.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:54.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=12537' '!=' 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:54.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:54.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:18:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_3 2024-09-14T07:18:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_3 ']' 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_3' 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:54.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:54.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:54.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:54.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:54.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:54.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:54.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:54.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:54.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_34 2024-09-14T07:18:54.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:54.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:54.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:54.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:54.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:54.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:54.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:54.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:54.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:54.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:54.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:54.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:54.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:54.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:54.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_3 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '' 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:54.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:54.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:54.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:54.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:54.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:55.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:55.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:55.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:55.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:55.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:55.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=12537' '!=' 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:55.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:55.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:18:55.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_3 2024-09-14T07:18:55.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:55.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:55.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_3 ']' 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_3' 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:55.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:55.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:55.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:55.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:55.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:55.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:55.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:55.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:55.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_34 2024-09-14T07:18:55.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:55.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:55.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:55.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:55.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:55.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:55.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:55.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:56.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:56.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_3 2024-09-14T07:18:56.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:56.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:56.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '' 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:56.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:56.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:56.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:56.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:56.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:56.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:56.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:56.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:56.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:56.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:56.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=12537' '!=' 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:56.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:56.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:18:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_3 2024-09-14T07:18:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:18:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:18:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:18:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_3 ']' 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_3' 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:18:57.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:18:57.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:18:57.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:18:57.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:57.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2024-09-14T07:18:57.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:18:57.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:57.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:18:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:18:57.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:18:57.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:18:57.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:57.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:57.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:18:57.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:57.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:18:57.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_34 2024-09-14T07:18:57.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:18:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:18:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2024-09-14T07:18:57.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:18:57.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:18:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:18:57.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:18:57.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:18:57.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:18:57.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:57.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:57.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:18:57.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2024-09-14T07:18:58.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:18:58.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_3 2024-09-14T07:18:58.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '' 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:18:58.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:58.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:58.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:58.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:58.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:58.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:58.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:58.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:58.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:58.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:18:58.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2024-09-14T07:18:58.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:18:58.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:18:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:18:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:18:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:18:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:18:58.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:18:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:18:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:18:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=12537], [object_number=8, tag_tid=1, entry_tid=12536], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:18:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:18:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=12537' '!=' 'object_number=9, tag_tid=1, entry_tid=12537' 2024-09-14T07:18:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:18:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_3 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_3 ']' 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_3' 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:00.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:00.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:00.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:00.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:00.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:00.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:00.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:00.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:00.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:00.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:00.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:00.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:00.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:00.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:00.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2024-09-14T07:19:00.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:00.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_34 2024-09-14T07:19:00.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:00.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2024-09-14T07:19:00.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:00.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_3 2024-09-14T07:19:00.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:00.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '' 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:00.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:19:00.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:19:00.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:00.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:19:00.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:19:00.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:19:00.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:19:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:19:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:19:00.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:19:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:19:00.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_3.status 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_3.status 2024-09-14T07:19:00.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:00.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:19:00.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2024-09-14T07:19:00.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:19:00.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:00.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:19:00.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:00.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:19:00.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=12535], [object_number=10, tag_tid=1, entry_tid=12534]]], state=connected] 2024-09-14T07:19:00.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.status 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:19:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:19:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:19:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:19:01.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2024-09-14T07:19:01.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_3': 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11d51c7cef0a 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11d51c7cef0a 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:09:25 2024 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:09:25 2024 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:27 2024 2024-09-14T07:19:01.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:19:01.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11d51c7cef0a 2024-09-14T07:19:01.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:19:01.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:19:01.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: b06b3cc7-22eb-4fa3-afba-9893bda59f01 2024-09-14T07:19:01.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:19:01.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_3 snap 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.export 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_3.export 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_3.export 2024-09-14T07:19:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.export 2024-09-14T07:19:01.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:19:01.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_3.export 2024-09-14T07:19:01.987 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:19:01.987 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:19:01.988 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:19:01.988 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:19:01.988 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:19:01.991 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:19:01.992 INFO:tasks.rbd_mirror_thrash:waiting for 93 secs before thrashing 2024-09-14T07:19:02.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:19:02.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_3.export 2024-09-14T07:19:02.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_3.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_3.export 2024-09-14T07:19:02.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:02.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:19:02.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_4 2024-09-14T07:19:02.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_4 snap 2024-09-14T07:19:02.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:02.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:02.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:02.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:02.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:19:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2024-09-14T07:19:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:19:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2024-09-14T07:19:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:19:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:19:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:19:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2024-09-14T07:19:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:19:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:19:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:19:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:19:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:19:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:04.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:19:04.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:04.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:19:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2024-09-14T07:19:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:19:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:19:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:19:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:19:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:19:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2024-09-14T07:19:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:19:04.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:19:04.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:19:04.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:19:04.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_4 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:04.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_4 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:04.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:19:05.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_4 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_4 ']' 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_4' 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2024-09-14T07:19:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:05.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:05.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:05.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:05.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:05.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:05.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2024-09-14T07:19:05.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_4 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '' 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:05.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:05.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:05.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:05.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:05.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:05.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:05.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11885' '!=' '' 2024-09-14T07:19:05.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:19:05.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11885' = '' 2024-09-14T07:19:05.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:05.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:19:05.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:05.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:19:05.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:19:05.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:05.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:19:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:19:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:05.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:19:05.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:19:05.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11885 2024-09-14T07:19:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11885 2024-09-14T07:19:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:19:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:19:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_4 2024-09-14T07:19:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_4 ']' 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_4' 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:05.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:05.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:05.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:05.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:05.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:05.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:05.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2024-09-14T07:19:05.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:05.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_4 2024-09-14T07:19:05.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '' 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:05.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:05.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:05.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:05.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:05.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:05.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:05.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:05.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:05.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:05.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:05.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:05.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11885' '!=' 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:05.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:05.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:19:06.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_4 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_4 ']' 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_4' 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2024-09-14T07:19:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:06.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:06.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2024-09-14T07:19:06.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_4 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '' 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:06.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:06.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:06.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:06.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:06.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:06.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:06.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2024-09-14T07:19:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:06.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:06.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:06.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:06.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:06.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:06.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:06.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11885' '!=' 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:19:07.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_4 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_4 ']' 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_4' 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:07.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:07.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:07.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:07.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:07.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:07.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:07.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:07.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2024-09-14T07:19:07.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_4 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '' 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:07.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:07.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:07.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:07.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:07.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:07.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:07.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:07.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:07.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:07.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:07.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:07.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:07.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:07.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:07.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:07.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11885' '!=' 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:07.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:07.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_4 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_4 ']' 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_4' 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:09.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:09.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:09.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:09.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:09.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:09.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:09.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:09.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2024-09-14T07:19:09.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:09.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_4 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '' 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:09.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:09.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:09.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:09.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:09.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:09.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:09.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:09.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:09.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:09.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:09.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11885], [object_number=8, tag_tid=1, entry_tid=11884], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:09.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:09.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:09.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:09.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11885' '!=' 'object_number=9, tag_tid=1, entry_tid=11885' 2024-09-14T07:19:09.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:09.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:19:11.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_4 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_4 ']' 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_4' 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:11.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:11.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:11.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:11.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:11.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:11.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:11.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:11.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:11.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2024-09-14T07:19:11.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_4 2024-09-14T07:19:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '' 2024-09-14T07:19:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:11.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:11.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:11.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:11.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:11.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:11.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:11.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:11.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2024-09-14T07:19:11.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:11.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_4.status 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_4.status 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:11.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2024-09-14T07:19:11.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:19:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:19:11.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:11.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:11.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11883], [object_number=10, tag_tid=1, entry_tid=11882]]], state=connected] 2024-09-14T07:19:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.status 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:19:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:19:12.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_4': 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed9be9cfab 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed9be9cfab 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:19:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:09:35 2024 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:09:35 2024 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:36 2024 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed9be9cfab 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 13a93793-2c8b-4755-b21e-e85eedcf468a 2024-09-14T07:19:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_4 snap 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.export 2024-09-14T07:19:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_4.export 2024-09-14T07:19:12.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_4.export 2024-09-14T07:19:12.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.export 2024-09-14T07:19:12.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:19:13.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_4.export 2024-09-14T07:19:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:19:13.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_4.export 2024-09-14T07:19:13.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_4.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_4.export 2024-09-14T07:19:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_5 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_5 snap 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:19:14.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:19:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:19:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2024-09-14T07:19:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:15.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:15.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:19:15.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:19:15.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:19:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:15.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:19:15.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:15.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:19:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2024-09-14T07:19:15.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:19:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:19:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:19:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:15.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:15.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2024-09-14T07:19:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:19:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:19:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:19:15.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:19:15.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:19:15.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:19:15.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_5 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_5 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:19:15.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:19:15.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:19:15.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:15.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_5 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_5 ']' 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_5' 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:16.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:16.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:16.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:16.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:16.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:16.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:16.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:16.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:16.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:16.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:16.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:16.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:16.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_5 2024-09-14T07:19:16.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '' 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:16.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:16.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:16.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:16.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2024-09-14T07:19:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:16.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:16.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:16.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:16.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:16.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:16.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:16.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6695' '!=' '' 2024-09-14T07:19:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:19:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6695' = '' 2024-09-14T07:19:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:16.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:19:16.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:16.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:19:16.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:19:16.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:19:16.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:16.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:19:16.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:16.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:19:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:19:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:19:16.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=6695 2024-09-14T07:19:16.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 6695 2024-09-14T07:19:16.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:19:16.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:16.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:19:16.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_5 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_5 ']' 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_5' 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:16.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:16.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:16.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:16.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:16.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:16.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:16.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_5 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '' 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:16.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:16.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:16.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:16.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:16.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:16.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6695' '!=' 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_5 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_5 ']' 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_5' 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:17.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:17.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:17.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:17.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:17.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_5 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '' 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:17.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:17.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:17.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:17.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:17.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:17.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:17.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:17.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:17.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6695' '!=' 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:17.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_5 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_5 ']' 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_5' 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:18.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:18.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:18.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:18.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:18.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:18.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:18.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:18.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:18.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:18.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:18.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:18.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_5 2024-09-14T07:19:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '' 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:18.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:18.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:18.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:18.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:18.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:18.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:18.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:18.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:18.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:18.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:18.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:18.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:18.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:18.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:18.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:18.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:18.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:18.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6695' '!=' 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_5 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_5 ']' 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_5' 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2024-09-14T07:19:20.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:20.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:20.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:20.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:20.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:20.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:20.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:20.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:20.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_5 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '' 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:20.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:20.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:20.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:20.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:20.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:20.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:20.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:20.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:20.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:20.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:20.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:20.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:20.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:20.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:20.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:20.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:20.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694], [object_number=5, tag_tid=1, entry_tid=6693], [object_number=4, tag_tid=1, entry_tid=6692]]], state=connected] 2024-09-14T07:19:20.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:20.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6695' '!=' 'object_number=7, tag_tid=1, entry_tid=6695' 2024-09-14T07:19:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:19:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:19:21.986 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:19:21.987 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:19:21.987 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T07:19:21.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:19:21.987+0000 7f366f242640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 90565) UID: 1000 2024-09-14T07:19:22.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:19:22.039+0000 7f3667a33640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55a1f7ffe700 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:19:22.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:19:22.039+0000 7f3667a33640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55a1f7ffe700 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_5 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_5 ']' 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_5' 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:19:22.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:22.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:22.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:22.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:22.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:22.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:22.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:22.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:22.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:22.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:22.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:22.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:22.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:22.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:22.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:22.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_50 2024-09-14T07:19:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:19:22.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:19:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:22.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:19:22.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:19:22.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:22.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:22.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:22.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:22.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:22.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:22.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2024-09-14T07:19:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:19:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:22.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:19:22.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:19:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:19:22.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:19:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:23.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:23.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:23.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:23.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:23.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:19:24.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:24.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:24.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:24.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:24.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:19:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:24.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:24.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:24.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:24.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:24.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_50 2024-09-14T07:19:24.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:24.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:24.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2024-09-14T07:19:24.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:19:24.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:24.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:24.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:24.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:19:24.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:19:24.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:19:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:24.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:24.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2024-09-14T07:19:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:19:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:19:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:19:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:24.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:24.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:19:26.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:26.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:26.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:26.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:26.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:26.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:26.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:26.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:26.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:26.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:26.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_50 2024-09-14T07:19:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2024-09-14T07:19:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:19:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:26.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:19:26.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:19:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:19:26.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:19:26.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:26.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:26.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:26.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:26.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:26.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:26.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:26.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:19:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:26.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:19:26.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:19:26.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:26.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:19:26.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:27.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:27.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:27.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:27.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 4 2024-09-14T07:19:27.987 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:19:27.987 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:19:27.988 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:19:27.988 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:19:27.988 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:19:27.991 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:19:27.991 INFO:tasks.rbd_mirror_thrash:waiting for 119 secs before thrashing 2024-09-14T07:19:31.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:31.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:31.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:31.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:31.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:31.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:31.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:31.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_50 2024-09-14T07:19:31.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:19:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:19:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:19:31.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:31.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:31.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:31.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:31.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:19:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:31.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:31.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:19:31.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:31.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:19:31.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:19:31.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:31.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:31.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:19:31.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:19:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:31.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:31.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:31.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:31.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:31.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:31.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:31.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:19:39.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:39.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:39.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:39.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:39.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:39.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:39.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:39.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:39.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:39.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:39.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:39.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:39.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:39.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:39.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_50 2024-09-14T07:19:39.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:39.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:39.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:19:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:19:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:19:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:19:39.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:40.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:40.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:19:40.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:19:40.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:19:40.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:19:40.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:40.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:40.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:40.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:19:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:48.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:48.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:48.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:48.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:48.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:48.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:19:48.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:48.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:48.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:19:48.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:19:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:19:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:19:48.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:48.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:48.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:19:48.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:no valid command found; 1 closest matches: 2024-09-14T07:19:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush mirror/image_50 2024-09-14T07:19:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:48.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:48.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:19:48.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:48.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:48.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:19:48.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:19:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:19:48.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:19:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:48.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:48.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:19:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:19:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:19:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:19:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:19:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-09-14T07:19:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:19:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:19:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:19:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:19:52.570 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:52.570+0000 7fb3eb959640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:19:52.570 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:52.570+0000 7fb3e9955640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:19:53.547 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:53.546+0000 7fb3e7150640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:19:53.555 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:53.554+0000 7fb3e694f640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:19:53.557 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:53.558+0000 7fb3ea957640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:19:53.689 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:53.690+0000 7fb3eb158640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:19:53.689 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:19:53.690+0000 7fb3ea957640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:19:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:19:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:19:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:19:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:19:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:19:56.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:19:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2024-09-14T07:19:56.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_5 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '' 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:56.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:19:56.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:56.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:56.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:56.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:56.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:56.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2024-09-14T07:19:57.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:19:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:19:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:19:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:19:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_5.status 2024-09-14T07:19:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_5.status 2024-09-14T07:19:57.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:19:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2024-09-14T07:19:57.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:19:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:19:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:19:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6695], [object_number=6, tag_tid=1, entry_tid=6694]]], state=connected] 2024-09-14T07:19:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.status 2024-09-14T07:19:57.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:57.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:19:57.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:57.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:19:57.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:19:58.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2024-09-14T07:19:58.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_5': 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1182d97b3129 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1182d97b3129 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:19:58.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:09:47 2024 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:09:47 2024 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:09:47 2024 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1182d97b3129 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 3bb9cf91-5d7a-40a8-a4f1-910b7a05cc33 2024-09-14T07:19:58.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_5 snap 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.export 2024-09-14T07:19:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_5.export 2024-09-14T07:19:58.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_5.export 2024-09-14T07:19:58.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.export 2024-09-14T07:19:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:19:58.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_5.export 2024-09-14T07:19:58.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:19:58.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_5.export 2024-09-14T07:19:58.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_5.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_5.export 2024-09-14T07:19:58.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_6 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_6 snap 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:19:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:01.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:01.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:01.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:01.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:02.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:02.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2024-09-14T07:20:02.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:02.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:02.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:02.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:02.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:02.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:02.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:02.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:02.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:02.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:02.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:02.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:02.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:02.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2024-09-14T07:20:02.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:02.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_6 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:02.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_6 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_6 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_6 ']' 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_6' 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:03.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:03.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:03.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:03.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:03.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:03.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2024-09-14T07:20:03.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_6 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '' 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:03.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:03.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:03.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14132], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130], [object_number=13, tag_tid=1, entry_tid=14129]]], state=connected] 2024-09-14T07:20:03.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2024-09-14T07:20:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:03.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:03.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:03.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:03.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14132], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130], [object_number=13, tag_tid=1, entry_tid=14129]]], state=connected] 2024-09-14T07:20:03.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14132' '!=' '' 2024-09-14T07:20:03.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:03.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14132' = '' 2024-09-14T07:20:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:20:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:20:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:03.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=14132 2024-09-14T07:20:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 14132 2024-09-14T07:20:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:03.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_6 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_6 ']' 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_6' 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:03.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:03.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:03.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:03.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:03.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:03.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:03.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:03.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2024-09-14T07:20:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_6 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '' 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:03.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:03.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:03.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:03.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:03.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:03.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14132], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130], [object_number=13, tag_tid=1, entry_tid=14129]]], state=connected] 2024-09-14T07:20:03.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:03.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:03.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:03.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:03.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:03.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:03.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:03.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:03.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:03.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:03.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:03.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14132], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130], [object_number=13, tag_tid=1, entry_tid=14129]]], state=connected] 2024-09-14T07:20:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:03.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14132' '!=' 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:03.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:03.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_6 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_6 ']' 2024-09-14T07:20:04.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_6' 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:04.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:04.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:04.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:04.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:04.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:04.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:04.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:04.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:04.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:04.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:04.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:04.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2024-09-14T07:20:04.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_6 2024-09-14T07:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '' 2024-09-14T07:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:04.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:04.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:04.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:04.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:04.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:04.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:04.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:04.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:04.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14132], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130], [object_number=13, tag_tid=1, entry_tid=14129]]], state=connected] 2024-09-14T07:20:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:04.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:04.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:04.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:04.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:04.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:04.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:04.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=12, tag_tid=1, entry_tid=14132], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130], [object_number=13, tag_tid=1, entry_tid=14129]]], state=connected] 2024-09-14T07:20:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:04.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:04.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:04.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=12, tag_tid=1, entry_tid=14132' '!=' 'object_number=12, tag_tid=1, entry_tid=14132' 2024-09-14T07:20:04.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:04.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:20:05.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_6 2024-09-14T07:20:05.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:05.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_6 ']' 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_6' 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:05.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:05.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:05.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:05.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2024-09-14T07:20:05.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:05.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:05.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:05.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:05.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2024-09-14T07:20:05.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_6 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '' 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:05.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:05.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:05.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:05.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:05.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:05.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:05.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:05.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2024-09-14T07:20:05.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:05.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:05.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:05.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2024-09-14T07:20:05.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:05.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:05.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:20:05.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:05.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_6.status 2024-09-14T07:20:05.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_6.status 2024-09-14T07:20:05.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2024-09-14T07:20:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:05.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:05.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:05.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:05.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:05.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=14131], [object_number=14, tag_tid=1, entry_tid=14130]]], state=connected] 2024-09-14T07:20:05.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.status 2024-09-14T07:20:05.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:05.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=2, entry_tid=1' -a 'object_number=13, tag_tid=2, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2024-09-14T07:20:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2024-09-14T07:20:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:06.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_6': 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b06dcdded9 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b06dcdded9 2024-09-14T07:20:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:09:54 2024 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:09:54 2024 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:55 2024 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b06dcdded9 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 4a03da9f-6715-4c6b-aa92-a7c84728e6bc 2024-09-14T07:20:06.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_6 snap 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.export 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_6.export 2024-09-14T07:20:06.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_6.export 2024-09-14T07:20:06.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.export 2024-09-14T07:20:07.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:07.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_6.export 2024-09-14T07:20:07.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:07.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_6.export 2024-09-14T07:20:07.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_6.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_6.export 2024-09-14T07:20:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_7 2024-09-14T07:20:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_7 snap 2024-09-14T07:20:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:07.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:07.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:07.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:07.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:07.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2024-09-14T07:20:07.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:07.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:07.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:08.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2024-09-14T07:20:08.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:08.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:08.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:08.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:08.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:08.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:08.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:08.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:08.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:08.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:08.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:08.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2024-09-14T07:20:08.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:08.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:08.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:08.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:08.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:08.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2024-09-14T07:20:08.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:08.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:08.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:08.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:08.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:08.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_7 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_7 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_7 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_7 ']' 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_7' 2024-09-14T07:20:09.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2024-09-14T07:20:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:09.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:09.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2024-09-14T07:20:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_7 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_7 '' 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_7.status 2024-09-14T07:20:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_7.status 2024-09-14T07:20:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2024-09-14T07:20:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2], [object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0]]], state=connected] 2024-09-14T07:20:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2024-09-14T07:20:09.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_7.status 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_7.status 2024-09-14T07:20:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:09.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2024-09-14T07:20:09.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:09.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:09.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:09.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:09.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2], [object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0]]], state=connected] 2024-09-14T07:20:09.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=15, tag_tid=2, entry_tid=3' 2024-09-14T07:20:09.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=15, tag_tid=2, entry_tid=3' 2024-09-14T07:20:09.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=3' '!=' '' 2024-09-14T07:20:09.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:09.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=15, tag_tid=2, entry_tid=3' = '' 2024-09-14T07:20:09.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=15, tag_tid=2, entry_tid=3' 2024-09-14T07:20:09.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=3, entry_tid=1' 2024-09-14T07:20:09.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:09.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:09.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:20:09.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=3' 2024-09-14T07:20:09.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:09.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:09.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T07:20:09.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=13, tag_tid=3, entry_tid=1' 2024-09-14T07:20:09.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:09.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:09.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:09.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=15, tag_tid=2, entry_tid=3' 2024-09-14T07:20:09.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:09.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:09.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=3 2024-09-14T07:20:09.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 3 2024-09-14T07:20:09.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:09.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:09.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_7 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_7 ']' 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_7' 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:09.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:09.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:09.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:09.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:09.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:09.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:09.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2024-09-14T07:20:09.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:09.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_7 2024-09-14T07:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_7 '' 2024-09-14T07:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_7.status 2024-09-14T07:20:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_7.status 2024-09-14T07:20:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2024-09-14T07:20:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:09.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:09.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:09.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:09.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:09.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_7.status 2024-09-14T07:20:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_7.status 2024-09-14T07:20:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2024-09-14T07:20:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 3 2024-09-14T07:20:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 3 2024-09-14T07:20:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=2, entry_tid=3], [object_number=14, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:09.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.status 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=13, tag_tid=3, entry_tid=1' 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=3, entry_tid=1' 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:09.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2024-09-14T07:20:11.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_7': 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11829858a04e 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11829858a04e 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:10:43 2024 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:10:43 2024 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:11:45 2024 2024-09-14T07:20:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:11.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11829858a04e 2024-09-14T07:20:11.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:11.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:11.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 306faee7-90b2-4c72-9856-ceb4a60d5ce4 2024-09-14T07:20:11.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:11.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:11.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_7 snap 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.export 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_7.export 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_7.export 2024-09-14T07:20:11.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.export 2024-09-14T07:20:11.134 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:20:11.134+0000 7fb3e9154640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:20:11.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:11.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_7.export 2024-09-14T07:20:11.511 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:20:11.512+0000 7fea4cb96640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:20:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:11.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_7.export 2024-09-14T07:20:11.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_7.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_7.export 2024-09-14T07:20:11.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:11.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_8 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_8 snap 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2024-09-14T07:20:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:15.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:15.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:15.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:15.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:15.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:15.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:15.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2024-09-14T07:20:15.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-09-14T07:20:15.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:20:15.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2024-09-14T07:20:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:20:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:20:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:20:15.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2024-09-14T07:20:16.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:16.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:16.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:16.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_8 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_8 2024-09-14T07:20:16.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_8 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_8 ']' 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_8' 2024-09-14T07:20:16.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:16.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:16.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:16.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2024-09-14T07:20:16.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:16.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:16.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:16.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:16.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:16.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:16.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:16.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-09-14T07:20:16.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:16.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:16.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:16.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2024-09-14T07:20:16.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:16.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:16.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:16.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:16.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:16.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:16.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:16.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:16.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:16.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:16.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:16.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:16.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_8 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '' 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:16.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:16.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:16.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:16.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:16.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:16.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:16.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:16.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:16.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:16.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:16.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9873' '!=' '' 2024-09-14T07:20:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9873' = '' 2024-09-14T07:20:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:16.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:16.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:16.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:16.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:20:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:16.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:20:16.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:16.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:16.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:16.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:16.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:16.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9873 2024-09-14T07:20:16.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9873 2024-09-14T07:20:16.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:16.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:16.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_8 2024-09-14T07:20:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_8 ']' 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_8' 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:16.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2024-09-14T07:20:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:16.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:16.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:16.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:16.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:16.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-09-14T07:20:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:17.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:17.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:17.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:17.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:17.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:17.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:17.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_8 2024-09-14T07:20:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '' 2024-09-14T07:20:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:17.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:17.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:17.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:17.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:17.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:17.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:17.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:17.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9873' '!=' 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:17.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:20:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_8 2024-09-14T07:20:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_8 ']' 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_8' 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:17.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:17.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-09-14T07:20:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:17.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:17.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:17.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:17.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:17.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:17.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:17.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:18.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_8 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '' 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:18.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:18.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:18.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:18.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:18.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:18.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:18.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:18.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:18.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:18.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:18.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2024-09-14T07:20:18.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:18.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:18.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:18.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:18.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:18.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:18.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:18.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:18.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:18.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:18.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9873' '!=' 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:18.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:18.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:20:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_8 2024-09-14T07:20:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_8 ']' 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_8' 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:18.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:18.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:19.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-09-14T07:20:19.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:19.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:19.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:19.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:19.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:19.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:19.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:19.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:19.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:19.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_8 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '' 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:19.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:19.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:19.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:19.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:19.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:19.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:19.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:19.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9873], [object_number=8, tag_tid=1, entry_tid=9872], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:19.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:19.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:19.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:19.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9873' '!=' 'object_number=9, tag_tid=1, entry_tid=9873' 2024-09-14T07:20:19.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:19.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:20:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_8 2024-09-14T07:20:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_8 ']' 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_8' 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:20.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:20.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:20.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:20.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:20.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2024-09-14T07:20:20.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:20.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:20.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:20.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:20.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:20.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:21.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:21.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:21.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:21.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-09-14T07:20:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:21.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2024-09-14T07:20:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:21.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:21.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:21.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:21.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:21.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:21.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:21.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:21.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2024-09-14T07:20:21.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_8 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '' 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:21.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:21.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:21.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:21.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:21.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:21.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:21.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:21.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:21.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_8.status 2024-09-14T07:20:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_8.status 2024-09-14T07:20:21.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:21.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:21.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2024-09-14T07:20:21.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9871], [object_number=10, tag_tid=1, entry_tid=9870]]], state=connected] 2024-09-14T07:20:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.status 2024-09-14T07:20:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2024-09-14T07:20:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:21.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2024-09-14T07:20:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_8': 2024-09-14T07:20:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:22.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123b43126ca6 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123b43126ca6 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:10:17 2024 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:10:17 2024 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:11:18 2024 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123b43126ca6 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:22.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:22.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 0454d4eb-d893-435f-b8c5-cdea72270cb7 2024-09-14T07:20:22.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:22.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:22.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_8 snap 2024-09-14T07:20:22.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:22.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:20:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.export 2024-09-14T07:20:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_8.export 2024-09-14T07:20:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_8.export 2024-09-14T07:20:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.export 2024-09-14T07:20:22.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:22.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_8.export 2024-09-14T07:20:22.905 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:20:22.904+0000 7fea4db98640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:20:22.906 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:20:22.904+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:20:23.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:23.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_8.export 2024-09-14T07:20:23.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_8.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_8.export 2024-09-14T07:20:23.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:23.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:23.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_9 2024-09-14T07:20:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_9 snap 2024-09-14T07:20:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2024-09-14T07:20:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:25.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2024-09-14T07:20:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:26.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:26.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2024-09-14T07:20:27.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:27.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:20:27.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:27.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:27.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:27.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:27.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:27.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:27.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:27.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:20:27.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2024-09-14T07:20:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:27.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:27.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:27.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:27.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_9 2024-09-14T07:20:27.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_9 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_9 2024-09-14T07:20:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_9 ']' 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_9' 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:27.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:27.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:27.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:27.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:27.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:27.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:27.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:27.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2024-09-14T07:20:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:27.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:27.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:27.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:27.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:27.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_9 2024-09-14T07:20:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '' 2024-09-14T07:20:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:27.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:27.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:27.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:27.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:27.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:27.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:27.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:27.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:27.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9009' '!=' '' 2024-09-14T07:20:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9009' = '' 2024-09-14T07:20:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:27.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:20:27.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:27.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:27.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:27.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:20:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:27.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9009 2024-09-14T07:20:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9009 2024-09-14T07:20:27.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:27.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:27.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_9 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_9 ']' 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_9' 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:27.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:27.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:27.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:28.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:28.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:28.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:28.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:28.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2024-09-14T07:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:28.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:28.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:28.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:28.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:28.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:28.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:28.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:28.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_9 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '' 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:28.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:28.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:28.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:28.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:28.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:28.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:28.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:28.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:28.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2024-09-14T07:20:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:28.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:28.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:28.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:28.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:28.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:28.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:28.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:28.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:28.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:28.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:28.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9009' '!=' 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:28.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:28.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:20:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_9 2024-09-14T07:20:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_9 ']' 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_9' 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:28.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:28.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:28.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:28.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:28.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:28.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:28.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:28.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:28.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:28.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:28.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:28.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:28.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:28.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:28.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:28.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:28.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:28.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:28.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:28.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:28.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:28.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:28.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:28.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:29.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_9 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '' 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:29.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:29.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:29.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:29.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:29.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:29.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:29.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:29.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:29.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:29.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:29.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:29.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2024-09-14T07:20:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2024-09-14T07:20:29.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:29.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:29.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:29.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:29.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:29.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9009' '!=' 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:29.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:20:29.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_9 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_9 ']' 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_9' 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:29.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:30.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:30.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:30.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:30.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:30.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:30.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:30.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:30.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:30.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:30.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:30.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:30.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:30.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:30.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:30.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:30.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:30.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:30.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:30.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:30.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:30.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:30.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:30.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:30.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:30.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_9 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '' 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:30.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:30.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:30.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:30.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:30.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:30.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:30.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:30.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:30.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:30.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2024-09-14T07:20:30.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:30.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:30.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:30.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2024-09-14T07:20:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:30.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:30.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:30.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:30.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:30.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:30.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9009' '!=' 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_9 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_9 ']' 2024-09-14T07:20:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_9' 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:32.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:32.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:32.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:32.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:32.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:32.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:32.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:32.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:32.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:32.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:32.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2024-09-14T07:20:32.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:32.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:32.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:32.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:32.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:32.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:32.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:32.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:32.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_9 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '' 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:32.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:32.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:32.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:32.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:32.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:32.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:32.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:32.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:32.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:32.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:32.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:32.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:32.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:32.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:32.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:32.364 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:20:32.366+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca54f26ea0 2024-09-14T07:20:32.365 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:20:32.366+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca54f27040 2024-09-14T07:20:32.365 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:20:32.366+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca55963930 2024-09-14T07:20:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9009], [object_number=8, tag_tid=1, entry_tid=9008], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:32.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:32.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:32.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:32.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9009' '!=' 'object_number=9, tag_tid=1, entry_tid=9009' 2024-09-14T07:20:32.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:32.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:20:34.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_9 2024-09-14T07:20:34.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:34.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_9 ']' 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_9' 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:34.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:34.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:34.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:34.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:34.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:34.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:34.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:34.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-09-14T07:20:34.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:34.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:34.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:34.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:34.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2024-09-14T07:20:34.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:34.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_9 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '' 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:34.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:34.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:34.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:34.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:34.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:34.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:34.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:34.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:34.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:20:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_9.status 2024-09-14T07:20:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_9.status 2024-09-14T07:20:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2024-09-14T07:20:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:34.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:20:34.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:20:34.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:34.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:34.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9007], [object_number=10, tag_tid=1, entry_tid=9006]]], state=connected] 2024-09-14T07:20:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.status 2024-09-14T07:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:34.992 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T07:20:34.993 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T07:20:34.993 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:20:34.994 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:20:34.998+0000 7fd59767e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 88933) UID: 1000 2024-09-14T07:20:35.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2024-09-14T07:20:35.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_9': 2024-09-14T07:20:35.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123b4be9a204 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123b4be9a204 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:10:26 2024 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:10:26 2024 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:26 2024 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123b4be9a204 2024-09-14T07:20:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: cee00132-8847-4ad7-ad34-a0946114ca65 2024-09-14T07:20:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:35.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:35.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_9 snap 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.export 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_9.export 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_9.export 2024-09-14T07:20:35.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.export 2024-09-14T07:20:36.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_9.export 2024-09-14T07:20:36.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:36.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_9.export 2024-09-14T07:20:36.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_9.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_9.export 2024-09-14T07:20:36.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_10 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_10 snap 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:36.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:38.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2024-09-14T07:20:39.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:39.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:39.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:39.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:39.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:39.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:39.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:39.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:39.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:39.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:39.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:39.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2024-09-14T07:20:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-14T07:20:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:20:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2024-09-14T07:20:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:20:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:39.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:20:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:20:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:39.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2024-09-14T07:20:40.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_10 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_10 2024-09-14T07:20:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_10 2024-09-14T07:20:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_10 ']' 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_10' 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-14T07:20:40.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:40.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:40.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:40.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_10 2024-09-14T07:20:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '' 2024-09-14T07:20:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:40.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:40.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:40.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:40.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:40.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:40.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:40.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:40.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:40.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:40.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:40.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:40.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2024-09-14T07:20:40.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:40.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:40.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=8190' '!=' '' 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=8190' = '' 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:20:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:20:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:40.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:40.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:40.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:40.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:40.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8190 2024-09-14T07:20:40.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 8190 2024-09-14T07:20:40.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:40.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:40.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_10 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_10 ']' 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_10' 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:40.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:40.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:40.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:40.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:40.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:40.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-14T07:20:40.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:40.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:40.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:40.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:40.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:40.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_10 2024-09-14T07:20:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '' 2024-09-14T07:20:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:41.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:41.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:41.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:41.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:41.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:41.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:41.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:41.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:41.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:41.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:41.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:41.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:41.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:41.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:41.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:41.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:41.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=8190' '!=' 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:41.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:41.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:20:41.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_10 2024-09-14T07:20:41.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:41.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:41.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_10 ']' 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_10' 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:41.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:41.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:41.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:41.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-14T07:20:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:41.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:41.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:41.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:41.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:41.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:41.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:41.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:41.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:41.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:41.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:41.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:41.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:41.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:41.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:41.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_10 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '' 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:41.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:41.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:41.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:41.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:42.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:42.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:42.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:42.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:42.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:42.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:42.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:42.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2024-09-14T07:20:42.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:42.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:42.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:42.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:42.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:42.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:42.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:42.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=8190' '!=' 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:42.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:42.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:20:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_10 2024-09-14T07:20:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_10 ']' 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_10' 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:42.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:42.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:43.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-14T07:20:43.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:43.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:43.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_10 2024-09-14T07:20:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '' 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:43.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2024-09-14T07:20:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:43.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:43.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:43.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:43.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:43.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=8190], [object_number=5, tag_tid=1, entry_tid=8189], [object_number=4, tag_tid=1, entry_tid=8188], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:43.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:43.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:43.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:43.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=8190' '!=' 'object_number=6, tag_tid=1, entry_tid=8190' 2024-09-14T07:20:43.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:43.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:20:43.993 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T07:20:43.994 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:20:43.994 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T07:20:43.994 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T07:20:43.995 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T07:20:43.998 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T07:20:43.998 INFO:tasks.rbd_mirror_thrash:waiting for 83 secs before thrashing 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_10 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_10 ']' 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_10' 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:44.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:44.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:44.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:44.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:44.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:44.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:44.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:44.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:44.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:44.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-14T07:20:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2024-09-14T07:20:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:45.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:45.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2024-09-14T07:20:45.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:45.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_10 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '' 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:45.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:45.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:45.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:45.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:45.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:45.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:45.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:45.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:45.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:45.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:45.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2024-09-14T07:20:45.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:45.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:45.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:45.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2024-09-14T07:20:45.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:45.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:45.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:20:45.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:45.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_10.status 2024-09-14T07:20:45.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_10.status 2024-09-14T07:20:45.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:45.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:45.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2024-09-14T07:20:45.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:45.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:45.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:45.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:45.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:45.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=8190], [object_number=7, tag_tid=1, entry_tid=8187]]], state=connected] 2024-09-14T07:20:45.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.status 2024-09-14T07:20:45.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:45.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:45.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:45.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:46.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_10': 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0992b56b2 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0992b56b2 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:10:35 2024 2024-09-14T07:20:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:10:35 2024 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:35 2024 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0992b56b2 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: e9f66bcf-f648-44dd-a72c-ed098f2998a6 2024-09-14T07:20:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_10 snap 2024-09-14T07:20:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:20:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.export 2024-09-14T07:20:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_10.export 2024-09-14T07:20:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_10.export 2024-09-14T07:20:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.export 2024-09-14T07:20:46.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_10.export 2024-09-14T07:20:46.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_10.export 2024-09-14T07:20:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_10.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_10.export 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_11 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_11 snap 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:47.084 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:20:47.081+0000 7fb3eb158640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:47.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:47.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:47.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:47.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:47.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:48.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2024-09-14T07:20:48.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:48.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:48.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:48.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:48.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:48.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:48.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2024-09-14T07:20:48.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:48.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:48.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:48.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:48.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_11 2024-09-14T07:20:48.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_11 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:48.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:49.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_11 2024-09-14T07:20:49.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_11 ']' 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_11' 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:49.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:49.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:49.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2024-09-14T07:20:49.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_11 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_11 '' 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:49.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_11.status 2024-09-14T07:20:49.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_11.status 2024-09-14T07:20:49.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:49.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2024-09-14T07:20:49.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:20:49.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:20:49.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:49.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:49.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2024-09-14T07:20:49.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:49.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2024-09-14T07:20:49.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_11.status 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_11.status 2024-09-14T07:20:49.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:49.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2024-09-14T07:20:49.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:20:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:20:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2024-09-14T07:20:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' '' 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' = '' 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:49.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:20:49.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:49.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:49.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T07:20:49.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:49.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:49.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:49.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:49.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:49.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=3 2024-09-14T07:20:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 3 2024-09-14T07:20:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:49.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_11 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_11 ']' 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_11' 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:49.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:49.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:49.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:49.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:49.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:49.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:49.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:49.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:49.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:49.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_11 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_11 '' 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:49.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_11.status 2024-09-14T07:20:49.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_11.status 2024-09-14T07:20:49.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2024-09-14T07:20:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:20:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:20:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2024-09-14T07:20:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2024-09-14T07:20:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_11.status 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_11.status 2024-09-14T07:20:49.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:49.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2024-09-14T07:20:49.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:20:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:20:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:49.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:49.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2024-09-14T07:20:49.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:49.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=3' '!=' 'object_number=3, tag_tid=2, entry_tid=3' 2024-09-14T07:20:49.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:49.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:20:50.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_11 2024-09-14T07:20:50.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:50.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:50.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:50.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_11 ']' 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_11' 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:50.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:50.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2024-09-14T07:20:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:50.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:50.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:50.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:50.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:50.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:50.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:50.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:50.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:50.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:50.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2024-09-14T07:20:50.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:50.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_11 2024-09-14T07:20:50.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:50.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_11 '' 2024-09-14T07:20:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:50.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:50.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_11.status 2024-09-14T07:20:50.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_11.status 2024-09-14T07:20:50.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:50.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:50.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2024-09-14T07:20:50.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:20:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:20:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:50.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:50.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2024-09-14T07:20:50.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:50.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:50.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_11.status 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_11.status 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:50.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:50.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2024-09-14T07:20:50.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:50.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:20:50.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:20:50.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:50.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:50.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:20:50.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.status 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:50.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2024-09-14T07:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_11': 2024-09-14T07:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed49069ace 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed49069ace 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:10:46 2024 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:10:46 2024 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:11:47 2024 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed49069ace 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 1655fa6b-584f-43de-b73c-caaecdd9954f 2024-09-14T07:20:51.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:51.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:51.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_11 snap 2024-09-14T07:20:51.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:51.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:20:51.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:51.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:51.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.export 2024-09-14T07:20:51.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_11.export 2024-09-14T07:20:51.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_11.export 2024-09-14T07:20:51.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.export 2024-09-14T07:20:51.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_11.export 2024-09-14T07:20:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:52.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_11.export 2024-09-14T07:20:52.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_11.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_11.export 2024-09-14T07:20:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_12 2024-09-14T07:20:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_12 snap 2024-09-14T07:20:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:20:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2024-09-14T07:20:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2024-09-14T07:20:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:20:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:20:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:20:54.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2024-09-14T07:20:54.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:20:54.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:20:54.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:54.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:54.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:20:54.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:20:54.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:54.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:20:54.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:54.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:20:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:54.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:54.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:54.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:20:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:54.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:20:54.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2024-09-14T07:20:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:20:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:54.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:54.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:20:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:20:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2024-09-14T07:20:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:55.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:20:55.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:55.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2024-09-14T07:20:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:20:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:55.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:20:55.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:20:55.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:20:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:55.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:55.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2024-09-14T07:20:55.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:55.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:20:55.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:55.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:55.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:55.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:20:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:20:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:20:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:20:55.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:20:55.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:55.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:55.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:55.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2024-09-14T07:20:55.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:20:55.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:20:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:20:55.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:20:55.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:20:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:20:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:20:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_12 2024-09-14T07:20:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:55.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_12 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_12 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_12 ']' 2024-09-14T07:20:55.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_12' 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:55.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:55.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:55.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:55.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:55.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2024-09-14T07:20:55.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:55.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:55.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:55.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:55.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:55.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:55.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:55.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:55.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:55.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:55.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:55.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:55.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:55.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:55.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:55.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:55.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:55.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:55.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:20:55.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:20:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:55.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:55.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:55.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:55.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:55.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_12 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:20:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:20:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:20:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:20:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:20:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:20:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:20:55.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:55.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:55.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:55.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_12 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_12 '' 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_12.status 2024-09-14T07:20:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_12.status 2024-09-14T07:20:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2024-09-14T07:20:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:56.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566], [object_number=5, tag_tid=1, entry_tid=7565], [object_number=4, tag_tid=1, entry_tid=7564]]], state=connected] 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:56.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_12.status 2024-09-14T07:20:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_12.status 2024-09-14T07:20:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2024-09-14T07:20:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:56.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566], [object_number=5, tag_tid=1, entry_tid=7565], [object_number=4, tag_tid=1, entry_tid=7564]]], state=connected] 2024-09-14T07:20:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7567' '!=' '' 2024-09-14T07:20:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:20:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7567' = '' 2024-09-14T07:20:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:56.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:56.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:56.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:56.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:20:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:20:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:56.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:20:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:56.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:20:56.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:56.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:20:56.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7567 2024-09-14T07:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7567 2024-09-14T07:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:56.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_12 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_12 ']' 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_12' 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:56.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:56.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:56.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:56.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:56.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:56.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:56.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2024-09-14T07:20:56.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:56.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:56.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:56.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:56.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:56.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:56.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:56.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:56.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:56.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:56.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:56.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:56.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:56.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2024-09-14T07:20:56.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:20:56.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:56.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:20:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:56.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:56.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:56.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:20:56.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:20:56.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:20:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:56.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:56.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:56.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:56.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:56.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_12 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:20:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:20:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:20:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:20:56.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:56.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:56.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:56.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:56.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:56.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_12 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_12 '' 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_12.status 2024-09-14T07:20:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_12.status 2024-09-14T07:20:56.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:56.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2024-09-14T07:20:56.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:57.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566], [object_number=5, tag_tid=1, entry_tid=7565], [object_number=4, tag_tid=1, entry_tid=7564]]], state=connected] 2024-09-14T07:20:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:57.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_12.status 2024-09-14T07:20:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_12.status 2024-09-14T07:20:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2024-09-14T07:20:57.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566], [object_number=5, tag_tid=1, entry_tid=7565], [object_number=4, tag_tid=1, entry_tid=7564]]], state=connected] 2024-09-14T07:20:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7567' '!=' 'object_number=7, tag_tid=1, entry_tid=7567' 2024-09-14T07:20:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:20:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:20:57.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_12 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_12 ']' 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_12' 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:57.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:20:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:20:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:20:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:20:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:20:57.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:57.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:20:57.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:57.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:57.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:20:57.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:20:57.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:20:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:20:57.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:20:57.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:57.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:57.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:20:57.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:57.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:57.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:57.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:57.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:57.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2024-09-14T07:20:57.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:20:57.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:57.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:57.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:57.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:20:57.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:20:57.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:57.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:57.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:57.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2024-09-14T07:20:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:20:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:20:57.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:20:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:20:57.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:20:57.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:57.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:57.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:20:57.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:57.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-09-14T07:20:57.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:20:57.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:20:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:20:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_12 2024-09-14T07:20:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:20:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:20:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:20:57.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:20:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:20:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:20:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:20:57.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:57.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:57.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:20:57.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_12 2024-09-14T07:20:58.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_12 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_12 '' 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:58.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:58.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:20:58.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_12.status 2024-09-14T07:20:58.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:58.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_12.status 2024-09-14T07:20:58.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:58.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2024-09-14T07:20:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:58.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:58.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:58.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_12.status 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_12.status 2024-09-14T07:20:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:20:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2024-09-14T07:20:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:58.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:20:58.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:20:58.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:20:58.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:58.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7567], [object_number=6, tag_tid=1, entry_tid=7566]]], state=connected] 2024-09-14T07:20:58.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.status 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:20:58.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:20:58.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:20:59.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2024-09-14T07:20:59.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_12': 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123bf82f9a6b 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123bf82f9a6b 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:10:51 2024 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:10:51 2024 2024-09-14T07:20:59.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:10:51 2024 2024-09-14T07:20:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:20:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123bf82f9a6b 2024-09-14T07:20:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:20:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:20:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 9cf66186-0f89-44da-8bfb-512f9292e134 2024-09-14T07:20:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:20:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_12 snap 2024-09-14T07:20:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:20:59.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:59.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:20:59.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.export 2024-09-14T07:20:59.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_12.export 2024-09-14T07:20:59.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_12.export 2024-09-14T07:20:59.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.export 2024-09-14T07:20:59.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:59.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_12.export 2024-09-14T07:20:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:20:59.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_12.export 2024-09-14T07:20:59.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_12.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_12.export 2024-09-14T07:20:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:20:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_13 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_13 snap 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:21:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2024-09-14T07:21:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:01.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:01.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:21:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:02.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:21:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:21:02.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:02.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2024-09-14T07:21:02.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:21:02.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:02.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:21:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:02.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:02.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:02.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:02.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2024-09-14T07:21:02.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:21:02.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:21:02.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:21:02.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:21:02.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:21:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:21:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_13 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_13 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:21:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:21:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_13 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_13 ']' 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_13' 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:03.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:03.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:03.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:03.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:03.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2024-09-14T07:21:03.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_13 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '' 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:03.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:03.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:03.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:03.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:03.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:03.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:03.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:03.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:03.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:03.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:03.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:03.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:03.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:03.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11609' '!=' '' 2024-09-14T07:21:03.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:21:03.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11609' = '' 2024-09-14T07:21:03.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:03.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:03.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:03.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:03.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:21:03.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:03.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:03.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:03.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:21:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:21:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:03.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11609 2024-09-14T07:21:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11609 2024-09-14T07:21:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_13 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_13 ']' 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_13' 2024-09-14T07:21:03.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:03.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:03.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:03.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:03.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:03.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:03.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:03.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2024-09-14T07:21:03.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_13 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '' 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:03.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:03.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:03.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:03.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:03.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:03.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:03.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:03.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:04.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:04.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:04.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:04.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:04.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11609' '!=' 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:21:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_13 2024-09-14T07:21:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:04.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_13 ']' 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_13' 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:04.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:04.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:04.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:04.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:04.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:04.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:04.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:04.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2024-09-14T07:21:04.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_13 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '' 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:04.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:04.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:04.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:04.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2024-09-14T07:21:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:04.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:04.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:04.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:04.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:04.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:04.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:04.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11609' '!=' 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:04.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:04.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_13 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_13 ']' 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_13' 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:05.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2024-09-14T07:21:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2024-09-14T07:21:05.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_13 2024-09-14T07:21:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:05.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '' 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:05.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:05.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:05.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:05.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:05.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:05.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:05.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:05.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:05.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:05.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:06.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:06.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2024-09-14T07:21:06.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:06.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:06.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:06.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:06.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:06.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:06.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:06.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:06.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:06.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:06.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:06.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:06.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:06.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11609' '!=' 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:06.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:06.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:21:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_13 2024-09-14T07:21:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_13 ']' 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_13' 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:07.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:07.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:07.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:07.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:07.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2024-09-14T07:21:07.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:07.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:07.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:07.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:07.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:07.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:07.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:07.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:07.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:07.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2024-09-14T07:21:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_13 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '' 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:08.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:08.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:08.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:08.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11609' '!=' 'object_number=9, tag_tid=1, entry_tid=11609' 2024-09-14T07:21:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:21:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_13 2024-09-14T07:21:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_13 ']' 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_13' 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:10.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:10.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:10.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:10.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2024-09-14T07:21:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_13 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '' 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:10.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:10.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:10.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:10.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:10.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:10.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:10.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:10.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:10.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:10.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_13.status 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_13.status 2024-09-14T07:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:10.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:10.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2024-09-14T07:21:10.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11607], [object_number=10, tag_tid=1, entry_tid=11606]]], state=connected] 2024-09-14T07:21:10.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.status 2024-09-14T07:21:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2024-09-14T07:21:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:10.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:10.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:10.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:10.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:10.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:21:10.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:11.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2024-09-14T07:21:11.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_13': 2024-09-14T07:21:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:21:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11825a5c8fd6 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11825a5c8fd6 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:00 2024 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:00 2024 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:12:01 2024 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11825a5c8fd6 2024-09-14T07:21:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:21:11.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:21:11.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 9f051053-923e-43cd-8832-0a8506372e9f 2024-09-14T07:21:11.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_13 snap 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.export 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_13.export 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_13.export 2024-09-14T07:21:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.export 2024-09-14T07:21:11.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:11.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_13.export 2024-09-14T07:21:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_13.export 2024-09-14T07:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_13.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_13.export 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_14 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_14 snap 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:12.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:12.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:21:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:13.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2024-09-14T07:21:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:21:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:21:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:21:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:21:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2024-09-14T07:21:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:21:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:13.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:21:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:21:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:13.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2024-09-14T07:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:21:13.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:13.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:13.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2024-09-14T07:21:13.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:13.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:13.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:13.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:21:13.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:21:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:13.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:13.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:13.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:13.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2024-09-14T07:21:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:21:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:21:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:14.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:14.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:14.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2024-09-14T07:21:14.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:21:14.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:21:14.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:21:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:21:14.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_14 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_14 2024-09-14T07:21:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_14 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_14 ']' 2024-09-14T07:21:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_14' 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:14.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:14.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:14.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:14.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:14.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:14.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:14.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:14.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:14.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:14.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:14.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:14.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:14.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:14.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:14.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2024-09-14T07:21:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:14.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:14.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:14.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_14 2024-09-14T07:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '' 2024-09-14T07:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:14.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:14.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:14.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:14.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:14.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:14.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:14.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:14.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:14.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:14.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:14.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' '' 2024-09-14T07:21:14.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:21:14.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' = '' 2024-09-14T07:21:14.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:21:14.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:14.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:14.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:14.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T07:21:14.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:14.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:14.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:14.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:21:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=1 2024-09-14T07:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 1 2024-09-14T07:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_14 2024-09-14T07:21:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_14 ']' 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_14' 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:15.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:15.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:15.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:15.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:15.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:15.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:15.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:15.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:15.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:15.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:15.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:15.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:15.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:15.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:15.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:15.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:15.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:15.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:15.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:15.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:15.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:15.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:15.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:15.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:15.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:15.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:15.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:15.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:15.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:15.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:15.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_14 2024-09-14T07:21:15.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:15.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:15.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:15.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '' 2024-09-14T07:21:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:15.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:15.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:15.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:15.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:15.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:15.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:15.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:15.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:15.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:15.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:15.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:15.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:15.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:15.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:15.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:15.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:15.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:15.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:15.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_14 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_14 ']' 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_14' 2024-09-14T07:21:16.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:16.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:16.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2024-09-14T07:21:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:16.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:16.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:16.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:16.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:16.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:16.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:16.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:16.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:16.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:16.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:16.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:16.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2024-09-14T07:21:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:16.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:16.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:16.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:16.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:16.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:16.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:16.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:16.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:16.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2024-09-14T07:21:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:16.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:16.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:16.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:16.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:16.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:16.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:16.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:16.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_14 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '' 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:16.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:16.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:16.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:16.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:16.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:16.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2024-09-14T07:21:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:16.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:16.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:16.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:16.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:16.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:21:17.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_14 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_14 ']' 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_14' 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:17.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:17.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:17.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:17.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:17.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:17.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:17.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:17.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:17.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:17.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:17.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:17.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2024-09-14T07:21:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:17.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:17.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:17.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:17.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:17.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:17.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:17.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:17.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:17.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:17.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:17.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:17.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:17.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:17.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:17.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:17.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_14 2024-09-14T07:21:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '' 2024-09-14T07:21:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:17.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:17.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:17.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:17.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:17.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:17.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:17.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:17.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:17.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:17.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:17.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:17.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:17.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:17.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:17.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:17.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:17.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:18.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:18.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:18.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:18.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:18.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:18.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:18.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:18.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:18.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:21:18.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:18.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_14 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_14 ']' 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_14' 2024-09-14T07:21:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:19.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:19.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:19.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:19.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2024-09-14T07:21:19.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:19.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:19.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:19.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:19.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:19.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:19.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:19.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:19.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:19.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:19.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:19.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:19.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:19.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-14T07:21:19.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:19.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:19.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:19.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:19.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:19.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:19.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2024-09-14T07:21:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_14 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '' 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:20.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:20.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:20.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:20.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:20.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:20.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:20.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:20.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:21:20.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:20.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_14.status 2024-09-14T07:21:20.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_14.status 2024-09-14T07:21:20.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:20.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:20.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2024-09-14T07:21:20.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:21:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:21:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3215], [object_number=2, tag_tid=1, entry_tid=3214]]], state=connected] 2024-09-14T07:21:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.status 2024-09-14T07:21:20.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:20.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:21:20.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:21:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:21.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_14': 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0a0e69bd6 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0a0e69bd6 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:21:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:14 2024 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:14 2024 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:12:15 2024 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0a0e69bd6 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: c243b721-22f8-40d5-9a92-bf5f590522a4 2024-09-14T07:21:21.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_14 snap 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.export 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_14.export 2024-09-14T07:21:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_14.export 2024-09-14T07:21:21.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.export 2024-09-14T07:21:21.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_14.export 2024-09-14T07:21:21.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:21.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_14.export 2024-09-14T07:21:21.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_14.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_14.export 2024-09-14T07:21:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:21.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_15 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_15 snap 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:21.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:21:23.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:23.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:21:24.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:21:24.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2024-09-14T07:21:24.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:21:24.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:21:24.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:24.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:24.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:21:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:21:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:21:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:24.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:24.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:24.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:24.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:24.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:21:24.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:21:24.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:24.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:24.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:24.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:24.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2024-09-14T07:21:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:21:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:21:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:21:24.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:24.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2024-09-14T07:21:25.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:25.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:21:25.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2024-09-14T07:21:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:21:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:25.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:25.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:25.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:21:25.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:21:25.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:25.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:25.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:25.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:25.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2024-09-14T07:21:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:21:25.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:21:25.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:21:25.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:21:25.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:21:25.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:21:25.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:21:25.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_15 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_15 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:21:25.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:25.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:25.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:25.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:25.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:25.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:25.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:25.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:25.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:25.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:25.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:25.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:25.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:25.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:25.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:25.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:25.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:25.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:25.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:25.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:25.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:25.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:25.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:25.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:25.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:25.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:25.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:25.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:25.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:25.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:25.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:25.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:25.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:25.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:25.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:25.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:25.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:25.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:25.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:25.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:25.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:25.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:25.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:25.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:25.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:25.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:25.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:25.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:25.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' '!=' '' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' = '' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:25.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:21:25.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:25.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:25.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:25.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:21:25.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:25.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:25.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:25.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:21:25.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:25.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:25.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:25.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7678 2024-09-14T07:21:25.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7678 2024-09-14T07:21:25.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:25.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:25.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:26.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:26.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:26.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:26.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:26.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:26.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:26.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:26.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:26.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:26.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:26.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:26.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:26.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:26.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:26.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:26.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:26.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:26.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:26.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:26.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:26.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:26.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:26.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:26.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' '!=' 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:26.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:26.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:21:26.992 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.292311772490479) > weight (0.25) 2024-09-14T07:21:26.992 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T07:21:26.993 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T07:21:26.993 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-14T07:21:26.994 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:21:26.996+0000 7f36ade17640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 69372) UID: 1000 2024-09-14T07:21:27.046 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:21:27.048+0000 7f36a6e09640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x563cfbcd3c00 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:21:27.046 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:21:27.048+0000 7f36a6e09640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x563cfbcd3c00 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:27.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:27.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:27.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:27.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:27.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:27.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:27.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:27.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:21:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:27.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:27.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:27.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:27.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:27.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:27.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:27.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:27.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:27.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:27.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:27.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:27.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:27.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:27.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:27.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:27.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:27.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:27.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:27.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:27.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:27.668 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' '!=' 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:27.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:28.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:28.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:28.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:28.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:28.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:28.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:28.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:28.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:28.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:28.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:28.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:28.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:28.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:28.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:28.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:28.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:28.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:28.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:28.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:28.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:28.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:28.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:28.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:28.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:28.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:28.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:28.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:28.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:28.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:28.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:28.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:28.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:28.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:28.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:28.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:28.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:28.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:28.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:28.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:28.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:28.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:28.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:28.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:28.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:28.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:28.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:28.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:28.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:29.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:29.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:29.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:29.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' '!=' 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:29.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:29.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:21:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:30.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:30.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:30.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:30.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:30.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:30.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:30.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:30.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:30.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:30.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:30.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:30.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:30.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:30.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:30.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:30.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:30.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:30.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:30.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:30.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:21:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:30.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:30.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:30.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:30.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:31.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:31.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:31.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:31.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:31.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:31.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:31.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:31.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:31.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:31.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:31.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:31.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:31.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:31.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:31.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' '!=' 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:31.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:31.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:21:32.993 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T07:21:32.994 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:21:32.994 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T07:21:32.994 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T07:21:32.995 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T07:21:32.997 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T07:21:32.998 INFO:tasks.rbd_mirror_thrash:waiting for 73 secs before thrashing 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:33.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:33.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:33.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:33.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:33.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:33.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:33.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:33.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:33.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:33.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:33.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:33.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:33.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:33.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:33.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:33.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:33.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:33.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:33.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:33.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:33.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:33.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:33.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:33.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:33.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:33.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:33.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:33.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:33.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:33.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7678], [object_number=5, tag_tid=1, entry_tid=7677], [object_number=4, tag_tid=1, entry_tid=7676], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:33.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7678' '!=' 'object_number=6, tag_tid=1, entry_tid=7678' 2024-09-14T07:21:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:33.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:21:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_15 2024-09-14T07:21:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:35.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_15 ']' 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_15' 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:35.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:35.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:35.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:35.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:35.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:35.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:35.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:35.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:35.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:35.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:35.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:35.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-09-14T07:21:35.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:35.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2024-09-14T07:21:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:35.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:35.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:35.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2024-09-14T07:21:36.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_15 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '' 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:36.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:36.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:36.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:36.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:36.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:36.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:36.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:36.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:36.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_15.status 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_15.status 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2024-09-14T07:21:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:36.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:36.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:36.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:36.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:36.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7678], [object_number=7, tag_tid=1, entry_tid=7675]]], state=connected] 2024-09-14T07:21:36.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.status 2024-09-14T07:21:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:21:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2024-09-14T07:21:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:21:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:37.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2024-09-14T07:21:37.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_15': 2024-09-14T07:21:37.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:21:37.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed7bcee8a9 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed7bcee8a9 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:21 2024 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:21 2024 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:11:21 2024 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed7bcee8a9 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:21:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:21:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: c33df5e7-01fc-4923-992c-364a014e17d4 2024-09-14T07:21:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:21:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_15 snap 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.export 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_15.export 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_15.export 2024-09-14T07:21:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.export 2024-09-14T07:21:37.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_15.export 2024-09-14T07:21:37.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_15.export 2024-09-14T07:21:38.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_15.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_15.export 2024-09-14T07:21:38.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_16 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_16 snap 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:38.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:21:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2024-09-14T07:21:38.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:38.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:21:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:21:39.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2024-09-14T07:21:39.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:21:39.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:21:39.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:39.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:39.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:21:39.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:21:39.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:39.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:21:39.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:39.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:39.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:39.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:39.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:21:39.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:39.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:39.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:21:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:39.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:39.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:39.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-09-14T07:21:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:21:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:21:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:21:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:21:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_16 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:39.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_16 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_16 2024-09-14T07:21:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_16 ']' 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_16' 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:40.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:40.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:40.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:40.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:40.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:40.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2024-09-14T07:21:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_16 2024-09-14T07:21:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_16 '' 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_16.status 2024-09-14T07:21:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_16.status 2024-09-14T07:21:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2024-09-14T07:21:40.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:40.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:40.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:40.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:40.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410], [object_number=5, tag_tid=1, entry_tid=5409], [object_number=4, tag_tid=1, entry_tid=5408]]], state=connected] 2024-09-14T07:21:40.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:40.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_16.status 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_16.status 2024-09-14T07:21:40.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:40.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2024-09-14T07:21:40.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:40.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:40.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:40.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:40.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410], [object_number=5, tag_tid=1, entry_tid=5409], [object_number=4, tag_tid=1, entry_tid=5408]]], state=connected] 2024-09-14T07:21:40.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=5411' '!=' '' 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=5411' = '' 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:40.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:21:40.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:40.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:21:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:21:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=5411 2024-09-14T07:21:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 1 -a 1 -le 5411 2024-09-14T07:21:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:40.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_16 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_16 ']' 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_16' 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:40.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:40.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:40.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:40.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:40.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:40.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:40.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2024-09-14T07:21:40.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:40.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2024-09-14T07:21:40.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_16 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_16 '' 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:40.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_16.status 2024-09-14T07:21:40.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_16.status 2024-09-14T07:21:40.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:40.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2024-09-14T07:21:40.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410], [object_number=5, tag_tid=1, entry_tid=5409], [object_number=4, tag_tid=1, entry_tid=5408]]], state=connected] 2024-09-14T07:21:40.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_16.status 2024-09-14T07:21:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_16.status 2024-09-14T07:21:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2024-09-14T07:21:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410], [object_number=5, tag_tid=1, entry_tid=5409], [object_number=4, tag_tid=1, entry_tid=5408]]], state=connected] 2024-09-14T07:21:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=5411' '!=' 'object_number=7, tag_tid=1, entry_tid=5411' 2024-09-14T07:21:40.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:40.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:21:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_16 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_16 ']' 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_16' 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:41.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2024-09-14T07:21:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_16 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_16 '' 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:41.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_16.status 2024-09-14T07:21:41.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_16.status 2024-09-14T07:21:41.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2024-09-14T07:21:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:41.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:41.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:41.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:41.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:41.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:41.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:41.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:41.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2024-09-14T07:21:41.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:41.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:41.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:41.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_16.status 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_16.status 2024-09-14T07:21:41.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:41.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:41.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2024-09-14T07:21:41.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:21:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:21:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:41.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:41.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5411], [object_number=6, tag_tid=1, entry_tid=5410]]], state=connected] 2024-09-14T07:21:41.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.status 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:41.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:21:41.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:42.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_16': 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123b13665b16 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123b13665b16 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:21:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:29 2024 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:29 2024 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:19:53 2024 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123b13665b16 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 3f4b61ee-0a9d-44c5-bb7c-5af088bec245 2024-09-14T07:21:42.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_16 snap 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.export 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_16.export 2024-09-14T07:21:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_16.export 2024-09-14T07:21:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.export 2024-09-14T07:21:42.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_16.export 2024-09-14T07:21:43.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_16.export 2024-09-14T07:21:43.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_16.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_16.export 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_17 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_17 snap 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:21:46.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2024-09-14T07:21:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:21:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:21:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:21:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:21:47.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:47.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:21:47.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:47.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:47.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:21:47.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:47.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:47.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:47.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:47.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2024-09-14T07:21:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:21:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2024-09-14T07:21:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:21:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:47.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:21:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:21:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:21:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:21:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2024-09-14T07:21:48.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:48.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:21:48.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:21:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:21:48.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:48.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:48.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:48.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:48.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2024-09-14T07:21:48.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:21:48.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:21:48.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:21:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:21:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_17 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_17 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:48.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_17 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_17 ']' 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_17' 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:48.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:48.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:48.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:48.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:48.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:48.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:48.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:48.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:48.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:48.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:48.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:48.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:48.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:48.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:48.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:48.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:48.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:48.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:48.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:48.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:48.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:48.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:48.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:48.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:48.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:48.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:48.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:48.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:48.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_17 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '' 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:48.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:48.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:48.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:48.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:48.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:48.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:48.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:48.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:48.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:49.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11833' '!=' '' 2024-09-14T07:21:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:21:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11833' = '' 2024-09-14T07:21:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:21:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:21:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:21:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:21:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:21:49.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:21:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11833 2024-09-14T07:21:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11833 2024-09-14T07:21:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:21:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_17 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_17 ']' 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_17' 2024-09-14T07:21:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:49.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:49.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:49.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:49.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:49.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:49.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:49.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:49.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:49.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2024-09-14T07:21:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:49.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:49.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:49.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:49.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:49.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:49.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:49.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:49.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:49.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:49.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:49.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_17 2024-09-14T07:21:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '' 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:49.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:49.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:49.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11833' '!=' 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:21:50.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_17 2024-09-14T07:21:50.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:50.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:50.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:50.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_17 ']' 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_17' 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:50.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:50.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:50.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:50.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:50.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:50.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:50.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:50.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:50.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:50.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:50.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:50.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:50.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:50.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:50.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:50.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:50.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:50.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:50.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:50.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:50.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:50.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_17 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '' 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:50.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:50.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:50.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:50.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:50.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:50.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:50.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:50.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:50.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:50.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:50.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:50.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:50.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:50.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:50.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11833' '!=' 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:21:51.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_17 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_17 ']' 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_17' 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:51.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:51.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:51.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:51.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:51.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:51.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:51.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:51.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:51.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:51.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:51.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:51.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:51.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2024-09-14T07:21:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:51.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_17 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '' 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:51.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:51.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:52.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:52.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:52.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:52.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:52.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:52.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11833' '!=' 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:21:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_17 2024-09-14T07:21:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:53.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_17 ']' 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_17' 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:53.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:53.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:53.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:53.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:53.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:53.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:53.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:53.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:53.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:53.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:53.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2024-09-14T07:21:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:53.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:53.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:53.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:53.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:53.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:53.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:53.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:53.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:53.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2024-09-14T07:21:53.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:53.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:53.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:53.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:53.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:53.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:53.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:54.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:54.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_17 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '' 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:54.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:54.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:54.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:54.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:54.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:54.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:54.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:54.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:54.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:54.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:54.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:54.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:54.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:54.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:54.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:54.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:54.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=11833], [object_number=8, tag_tid=1, entry_tid=11832], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:54.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:54.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:54.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:54.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=11833' '!=' 'object_number=9, tag_tid=1, entry_tid=11833' 2024-09-14T07:21:54.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:21:54.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:21:56.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_17 2024-09-14T07:21:56.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:56.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:56.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_17 ']' 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_17' 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:21:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:21:56.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:21:56.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:21:56.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:21:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:21:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:21:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:21:56.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:56.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2024-09-14T07:21:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:21:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:21:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:56.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:56.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:21:56.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:21:56.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:21:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:21:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:56.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-14T07:21:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:21:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_17 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:21:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:21:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:21:56.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:21:56.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:56.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:56.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:21:56.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_17 2024-09-14T07:21:56.652 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:21:56.651+0000 7fc0873cc640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55755d19ea80 get_replay_status: replay not running 2024-09-14T07:21:56.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:56.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_17 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '' 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:56.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:56.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:56.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:56.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:56.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:56.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:56.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2024-09-14T07:21:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_17.status 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_17.status 2024-09-14T07:21:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:21:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2024-09-14T07:21:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:21:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:21:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:21:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:57.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11831], [object_number=10, tag_tid=1, entry_tid=11830]]], state=connected] 2024-09-14T07:21:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.status 2024-09-14T07:21:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:21:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2024-09-14T07:21:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:21:57.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:57.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:57.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:57.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:21:57.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:21:57.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:21:57.642 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:21:57.643+0000 7fb3ea156640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:21:57.709 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:21:57.643+0000 7fb3e8152640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 8 2024-09-14T07:21:57.709 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:21:57.655+0000 7fb3e614e640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 7 2024-09-14T07:21:57.709 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:21:57.655+0000 7fb3e7150640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:21:57.709 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:21:57.703+0000 7fb3eb158640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 2 2024-09-14T07:21:58.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_17': 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11821b488587 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11821b488587 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:21:58.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:35 2024 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:35 2024 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:12:36 2024 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11821b488587 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: c506064a-1399-4f59-b610-2c30b923deb6 2024-09-14T07:21:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:21:58.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_17 snap 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.export 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_17.export 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_17.export 2024-09-14T07:21:58.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.export 2024-09-14T07:21:58.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:58.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_17.export 2024-09-14T07:21:58.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:21:58.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_17.export 2024-09-14T07:21:58.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_17.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_17.export 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_18 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_18 snap 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:21:58.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:21:58.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:21:58.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:00.410 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:22:00.411+0000 7fb3e414a640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:22:00.463 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:22:00.463+0000 7fb3e694f640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2024-09-14T07:22:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2024-09-14T07:22:02.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:02.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:02.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:02.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2024-09-14T07:22:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:03.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:22:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:22:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2024-09-14T07:22:03.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:03.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:03.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_18 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:03.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_18 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_18 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_18 ']' 2024-09-14T07:22:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_18' 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:03.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:03.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:03.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:03.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:03.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:03.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:03.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:03.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:03.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:03.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:03.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:03.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:03.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:03.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:03.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:03.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:03.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:03.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:03.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:03.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:03.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:03.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:03.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:03.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:03.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:03.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:03.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:03.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:03.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:03.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:03.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_18 2024-09-14T07:22:03.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:03.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:03.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:03.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '' 2024-09-14T07:22:03.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:03.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:03.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:03.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:03.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:03.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:03.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:03.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:03.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:03.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:03.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:03.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:03.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:03.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:03.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:03.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:03.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:03.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:03.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:03.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:03.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:03.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7226' '!=' '' 2024-09-14T07:22:03.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:03.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7226' = '' 2024-09-14T07:22:03.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:03.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:03.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:03.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:03.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:22:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:03.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:03.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:03.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:03.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:03.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:03.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:03.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7226 2024-09-14T07:22:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7226 2024-09-14T07:22:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_18 2024-09-14T07:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_18 ']' 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_18' 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:04.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:04.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:04.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:04.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:04.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:04.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:04.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:04.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:04.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:04.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:04.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:04.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:04.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2024-09-14T07:22:04.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:04.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:04.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:04.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_18 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '' 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:04.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:04.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:04.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:04.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:04.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:04.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:04.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:04.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:04.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:04.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:04.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:04.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:04.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:04.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7226' '!=' 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:04.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:04.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:22:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_18 2024-09-14T07:22:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_18 ']' 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_18' 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:05.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2024-09-14T07:22:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:05.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:05.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:05.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:05.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:05.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:05.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:05.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:05.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:05.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:05.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:05.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:05.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:05.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:05.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:05.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:05.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:05.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:05.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:05.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:05.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:05.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_18 2024-09-14T07:22:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '' 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:05.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:05.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:05.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:05.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:05.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:05.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:05.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:05.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:05.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:05.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2024-09-14T07:22:05.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:05.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:05.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:05.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:05.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:05.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7226' '!=' 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:05.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:05.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:22:06.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_18 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_18 ']' 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_18' 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:06.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:06.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:06.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:06.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:06.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:06.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:06.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:06.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:06.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:06.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:06.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:06.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:06.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:06.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:06.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:06.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:06.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:06.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:06.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2024-09-14T07:22:06.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:06.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:06.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:06.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:06.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:06.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:06.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:06.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:06.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:06.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:06.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:06.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:06.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:06.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:06.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:06.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:06.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_18 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '' 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:06.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:06.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:06.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:06.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:06.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:06.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2024-09-14T07:22:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:06.999 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6252309016343142) > weight (0.25) 2024-09-14T07:22:06.999 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.42228652810257605) > weight (0.25) 2024-09-14T07:22:06.999 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:22:06.999 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:22:06.999 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T07:22:07.000 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:22:06.999+0000 7f7775213640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 94793) UID: 1000 2024-09-14T07:22:07.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:07.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:07.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:07.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:07.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:07.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7226' '!=' 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:22:08.001 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:22:08.001 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:22:08.001 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:22:08.001 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:22:08.002 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:22:08.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:22:08.005 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-09-14T07:22:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_18 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_18 ']' 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_18' 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:08.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:08.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:08.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:08.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:08.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:08.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:08.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2024-09-14T07:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:08.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:08.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2024-09-14T07:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:09.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:09.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:09.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:09.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:09.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:09.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_18 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '' 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:09.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:09.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:09.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:09.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:09.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:09.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:09.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7226], [object_number=5, tag_tid=1, entry_tid=7225], [object_number=4, tag_tid=1, entry_tid=7224], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:09.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:09.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:09.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7226' '!=' 'object_number=6, tag_tid=1, entry_tid=7226' 2024-09-14T07:22:09.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:09.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:22:11.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_18 2024-09-14T07:22:11.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_18 ']' 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_18' 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:11.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:11.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:11.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:11.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:11.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:11.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:11.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:11.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:11.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:11.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:11.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:11.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:11.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:11.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-14T07:22:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2024-09-14T07:22:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:11.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:11.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:11.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:11.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2024-09-14T07:22:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_18 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '' 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:11.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:11.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:11.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:11.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:11.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:11.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:11.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:11.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:11.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_18.status 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_18.status 2024-09-14T07:22:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:11.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:11.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2024-09-14T07:22:11.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7226], [object_number=7, tag_tid=1, entry_tid=7223]]], state=connected] 2024-09-14T07:22:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.status 2024-09-14T07:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2024-09-14T07:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:22:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:12.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2024-09-14T07:22:13.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_18': 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0533d9274 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0533d9274 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:47 2024 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:47 2024 2024-09-14T07:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:11:47 2024 2024-09-14T07:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0533d9274 2024-09-14T07:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 3a6a361c-afda-4451-826d-eeb995f6a4df 2024-09-14T07:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_18 snap 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.export 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_18.export 2024-09-14T07:22:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_18.export 2024-09-14T07:22:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.export 2024-09-14T07:22:13.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:13.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_18.export 2024-09-14T07:22:13.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:13.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_18.export 2024-09-14T07:22:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_18.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_18.export 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_19 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_19 snap 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2024-09-14T07:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2024-09-14T07:22:15.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:16.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:16.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2024-09-14T07:22:16.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:16.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:16.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:16.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2024-09-14T07:22:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2024-09-14T07:22:16.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T07:22:16.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:16.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:16.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:16.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2024-09-14T07:22:16.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:16.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:16.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:16.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:16.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2024-09-14T07:22:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:17.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_19 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_19 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:17.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:17.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_19 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_19 ']' 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_19' 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2024-09-14T07:22:17.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:17.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:17.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:17.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:17.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:17.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2024-09-14T07:22:17.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T07:22:17.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:17.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:17.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:17.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:17.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:17.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:17.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:17.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:17.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:17.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:17.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:17.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:17.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2024-09-14T07:22:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_19 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_19 '' 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:17.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_19.status 2024-09-14T07:22:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_19.status 2024-09-14T07:22:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:17.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:17.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2024-09-14T07:22:17.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=5264], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262], [object_number=5, tag_tid=1, entry_tid=5261]]], state=connected] 2024-09-14T07:22:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_19.status 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_19.status 2024-09-14T07:22:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2024-09-14T07:22:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:17.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:17.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:17.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:17.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=5264], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262], [object_number=5, tag_tid=1, entry_tid=5261]]], state=connected] 2024-09-14T07:22:17.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=5264' '!=' '' 2024-09-14T07:22:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=5264' = '' 2024-09-14T07:22:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:22:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:17.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:17.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:17.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:17.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=5264 2024-09-14T07:22:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 5264 2024-09-14T07:22:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_19 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_19 ']' 2024-09-14T07:22:17.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_19' 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:17.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:17.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:17.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:17.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:17.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:17.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:17.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2024-09-14T07:22:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T07:22:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2024-09-14T07:22:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:17.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2024-09-14T07:22:18.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_19 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_19 '' 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_19.status 2024-09-14T07:22:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_19.status 2024-09-14T07:22:18.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2024-09-14T07:22:18.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:18.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=5264], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262], [object_number=5, tag_tid=1, entry_tid=5261]]], state=connected] 2024-09-14T07:22:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2024-09-14T07:22:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_19.status 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_19.status 2024-09-14T07:22:18.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:18.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2024-09-14T07:22:18.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:18.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:18.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:18.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:18.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=4, tag_tid=1, entry_tid=5264], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262], [object_number=5, tag_tid=1, entry_tid=5261]]], state=connected] 2024-09-14T07:22:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:18.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:18.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=4, tag_tid=1, entry_tid=5264' '!=' 'object_number=4, tag_tid=1, entry_tid=5264' 2024-09-14T07:22:18.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:18.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:22:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_19 2024-09-14T07:22:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_19 ']' 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_19' 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:18.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:18.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:18.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:18.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:18.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2024-09-14T07:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-14T07:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:18.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:18.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:18.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:18.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:18.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:18.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2024-09-14T07:22:18.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_19 2024-09-14T07:22:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_19 '' 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_19.status 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:18.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_19.status 2024-09-14T07:22:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:18.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2024-09-14T07:22:18.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:19.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:19.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_19.status 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_19.status 2024-09-14T07:22:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:19.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:19.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2024-09-14T07:22:19.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:19.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:22:19.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:22:19.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5263], [object_number=6, tag_tid=1, entry_tid=5262]]], state=connected] 2024-09-14T07:22:19.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.status 2024-09-14T07:22:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:22:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2024-09-14T07:22:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:19.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:22:19.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:20.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_19': 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed799f18a1 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed799f18a1 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:22:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:11:59 2024 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:11:59 2024 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:11:59 2024 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed799f18a1 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: e7923553-ecfd-45e7-8575-0e25b064ffd7 2024-09-14T07:22:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:22:20.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:20.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_19 snap 2024-09-14T07:22:20.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:20.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:22:20.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:20.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:22:20.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.export 2024-09-14T07:22:20.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_19.export 2024-09-14T07:22:20.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_19.export 2024-09-14T07:22:20.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.export 2024-09-14T07:22:20.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_19.export 2024-09-14T07:22:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:20.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_19.export 2024-09-14T07:22:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_19.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_19.export 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_20 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_20 snap 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:23.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:24.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2024-09-14T07:22:24.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T07:22:24.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:24.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:24.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:24.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:24.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:24.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2024-09-14T07:22:25.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:25.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:25.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:25.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:25.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_20 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:25.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_20 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:25.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:25.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_20 2024-09-14T07:22:25.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_20 ']' 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_20' 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:25.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:25.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:25.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:25.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:25.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:25.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:25.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:25.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:25.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:25.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:25.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:25.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T07:22:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:25.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:25.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:25.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:25.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:25.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:25.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:25.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:25.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_20 2024-09-14T07:22:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:25.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '' 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:25.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:25.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:25.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:25.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:25.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:25.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:25.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:25.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:25.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:25.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:25.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9024' '!=' '' 2024-09-14T07:22:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9024' = '' 2024-09-14T07:22:25.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:25.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:25.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:25.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:25.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:22:25.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:25.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:25.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:25.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:25.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:25.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:25.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:25.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:25.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:25.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9024 2024-09-14T07:22:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9024 2024-09-14T07:22:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:25.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_20 2024-09-14T07:22:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:25.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_20 ']' 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_20' 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:25.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:25.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:25.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:25.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:25.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:25.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T07:22:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:26.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:26.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:26.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:26.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:26.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:26.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:26.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:26.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:26.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:26.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_20 2024-09-14T07:22:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '' 2024-09-14T07:22:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:26.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:26.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:26.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:26.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:26.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:26.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:26.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:26.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:26.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:26.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:26.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2024-09-14T07:22:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2024-09-14T07:22:26.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:26.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:26.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:26.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:26.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:26.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9024' '!=' 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:26.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:26.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:22:26.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_20 2024-09-14T07:22:26.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:26.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_20 ']' 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_20' 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:26.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:26.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2024-09-14T07:22:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:26.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:26.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T07:22:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:26.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:26.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:26.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:26.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:26.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:27.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:27.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_20 2024-09-14T07:22:27.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '' 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:27.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:27.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:27.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2024-09-14T07:22:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:27.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:27.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:27.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:27.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:27.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:27.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:27.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:27.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:27.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:27.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:27.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9024' '!=' 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:27.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:27.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:22:27.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_20 2024-09-14T07:22:27.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:27.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:27.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_20 ']' 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_20' 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2024-09-14T07:22:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:27.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:27.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:27.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:27.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:27.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:27.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:28.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T07:22:28.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:28.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2024-09-14T07:22:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:28.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:28.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:28.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:28.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:28.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:28.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_20 2024-09-14T07:22:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '' 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:28.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:28.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:28.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:28.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:28.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:28.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:28.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:28.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:28.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:28.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2024-09-14T07:22:28.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:28.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9024], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022], [object_number=9, tag_tid=1, entry_tid=9021]]], state=connected] 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9024' '!=' 'object_number=8, tag_tid=1, entry_tid=9024' 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:28.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_20 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_20 ']' 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_20' 2024-09-14T07:22:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2024-09-14T07:22:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:29.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:29.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:29.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:29.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:29.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:30.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-14T07:22:30.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:30.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:30.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:30.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:30.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:30.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:30.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:30.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2024-09-14T07:22:30.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:30.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_20 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '' 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:30.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:30.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:30.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:30.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:30.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:22:30.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_20.status 2024-09-14T07:22:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_20.status 2024-09-14T07:22:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2024-09-14T07:22:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:30.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9023], [object_number=10, tag_tid=1, entry_tid=9022]]], state=connected] 2024-09-14T07:22:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.status 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:30.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:30.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:30.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:22:30.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2024-09-14T07:22:31.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_20': 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123bbd24ff14 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123bbd24ff14 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:12:06 2024 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:12:06 2024 2024-09-14T07:22:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:13:07 2024 2024-09-14T07:22:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:22:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123bbd24ff14 2024-09-14T07:22:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:22:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:22:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 4abdf60c-b588-4ab1-94c6-7337e96e901f 2024-09-14T07:22:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:22:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_20 snap 2024-09-14T07:22:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:22:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:22:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.export 2024-09-14T07:22:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_20.export 2024-09-14T07:22:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_20.export 2024-09-14T07:22:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.export 2024-09-14T07:22:31.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_20.export 2024-09-14T07:22:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_20.export 2024-09-14T07:22:31.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_20.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_20.export 2024-09-14T07:22:32.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_21 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_21 snap 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:32.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:32.093 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:22:32.090+0000 7fb3e494b640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:22:32.094 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:22:32.094+0000 7fb3e694f640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:22:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2024-09-14T07:22:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2024-09-14T07:22:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2024-09-14T07:22:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:33.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:33.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:33.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:33.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:33.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2024-09-14T07:22:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2024-09-14T07:22:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:33.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:33.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:33.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:33.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:33.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2024-09-14T07:22:34.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:34.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:34.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:34.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:34.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_21 2024-09-14T07:22:34.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:22:34.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:34.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:34.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:22:34.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:22:34.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:34.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:34.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:34.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:34.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_21 2024-09-14T07:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:34.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:34.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:34.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:34.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_21 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_21 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:34.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:34.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:34.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:34.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_21 2024-09-14T07:22:34.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:34.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:34.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_21 ']' 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_21' 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:34.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:34.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2024-09-14T07:22:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:34.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:34.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:34.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:34.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:34.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:34.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:34.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:34.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:34.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:34.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:34.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:34.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:34.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:34.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:34.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:34.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:34.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:34.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:34.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:34.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:34.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_21 2024-09-14T07:22:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '' 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3347], [object_number=2, tag_tid=1, entry_tid=3346], [object_number=1, tag_tid=1, entry_tid=3345], [object_number=0, tag_tid=1, entry_tid=3344]]], state=connected] 2024-09-14T07:22:34.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:34.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:34.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3347], [object_number=2, tag_tid=1, entry_tid=3346], [object_number=1, tag_tid=1, entry_tid=3345], [object_number=0, tag_tid=1, entry_tid=3344]]], state=connected] 2024-09-14T07:22:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3347' '!=' '' 2024-09-14T07:22:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3347' = '' 2024-09-14T07:22:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:22:34.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:34.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:34.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=3347 2024-09-14T07:22:35.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 1 -a 1 -le 3347 2024-09-14T07:22:35.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:35.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:35.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_21 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_21 ']' 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_21' 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:35.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:35.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2024-09-14T07:22:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:35.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:35.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:35.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:35.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:35.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:35.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:35.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:35.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:35.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:35.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:35.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:35.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:35.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:35.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:35.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:35.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:35.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:35.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2024-09-14T07:22:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:35.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:35.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:35.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:35.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:35.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:35.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:35.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_21 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '' 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:35.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:35.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:35.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:35.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:35.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:35.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:35.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:35.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:35.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=3633], [object_number=0, tag_tid=1, entry_tid=3632], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:35.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:35.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:35.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=3633], [object_number=0, tag_tid=1, entry_tid=3632], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:35.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:35.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:35.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:35.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=3633' '!=' 'object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=3633' = 'object_number=3, tag_tid=1, entry_tid=3347' 2024-09-14T07:22:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:22:35.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:35.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:35.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:35.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:35.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:35.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:35.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:35.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:35.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:35.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=3633 2024-09-14T07:22:35.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 1 -a 1 -le 3633 2024-09-14T07:22:35.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:35.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_21 2024-09-14T07:22:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_21 ']' 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_21' 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:35.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:35.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:36.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:36.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:36.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2024-09-14T07:22:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:36.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:36.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:36.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:36.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:36.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:36.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:36.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2024-09-14T07:22:36.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:36.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:36.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:36.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:36.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:36.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:36.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:36.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:36.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:36.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:36.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:36.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:36.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:36.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:36.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:36.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:36.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:36.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:36.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:36.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:36.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:36.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_21 2024-09-14T07:22:36.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:36.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:36.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:36.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '' 2024-09-14T07:22:36.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:36.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:36.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:36.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:36.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:36.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=3633], [object_number=0, tag_tid=1, entry_tid=3632], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:36.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:36.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:36.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:36.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:36.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=3633], [object_number=0, tag_tid=1, entry_tid=3632], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:36.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=3633' '!=' 'object_number=1, tag_tid=1, entry_tid=3633' 2024-09-14T07:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:22:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_21 2024-09-14T07:22:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_21 ']' 2024-09-14T07:22:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_21' 2024-09-14T07:22:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2024-09-14T07:22:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2024-09-14T07:22:36.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:36.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:36.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:36.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:36.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_21 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:37.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:37.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:37.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:37.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-14T07:22:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:37.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:37.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_21 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:37.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:37.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:37.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:37.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_21 2024-09-14T07:22:37.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_21 2024-09-14T07:22:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '' 2024-09-14T07:22:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:37.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:37.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:37.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:37.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:37.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:37.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:37.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:37.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2024-09-14T07:22:37.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_21.status 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_21.status 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:37.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:37.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2024-09-14T07:22:37.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:22:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:22:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3631], [object_number=2, tag_tid=1, entry_tid=3630]]], state=connected] 2024-09-14T07:22:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.status 2024-09-14T07:22:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:22:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:22:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:38.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_21': 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11824c5bb85 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11824c5bb85 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:22:38.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:12:17 2024 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:12:17 2024 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:53 2024 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11824c5bb85 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 8a7882da-4c7f-4fee-86e7-c24662d6ce25 2024-09-14T07:22:38.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_21 snap 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.export 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_21.export 2024-09-14T07:22:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_21.export 2024-09-14T07:22:38.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.export 2024-09-14T07:22:38.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:38.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_21.export 2024-09-14T07:22:38.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:38.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_21.export 2024-09-14T07:22:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_21.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_21.export 2024-09-14T07:22:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:22:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_22 2024-09-14T07:22:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_22 snap 2024-09-14T07:22:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:39.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:39.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:39.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:41.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2024-09-14T07:22:41.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:41.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:41.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:41.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:41.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:42.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2024-09-14T07:22:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2024-09-14T07:22:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:42.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:42.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2024-09-14T07:22:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:42.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:42.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:42.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:42.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:42.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:42.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:42.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:42.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2024-09-14T07:22:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T07:22:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2024-09-14T07:22:43.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T07:22:43.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:43.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_22 2024-09-14T07:22:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:22:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:43.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:22:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:43.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:43.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:43.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:43.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_22 2024-09-14T07:22:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:43.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:43.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:43.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_22 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_22 2024-09-14T07:22:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:43.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:43.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_22 2024-09-14T07:22:43.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:43.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:43.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_22 ']' 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_22' 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:43.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:43.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:43.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:43.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:43.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:43.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:43.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2024-09-14T07:22:43.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T07:22:43.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:43.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:43.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2024-09-14T07:22:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T07:22:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2024-09-14T07:22:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:43.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:43.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:43.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:43.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_22 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_22 '' 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_22.status 2024-09-14T07:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_22.status 2024-09-14T07:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2024-09-14T07:22:43.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:43.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386]]], state=connected] 2024-09-14T07:22:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9388], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386], [object_number=9, tag_tid=1, entry_tid=9385]]], state=connected] 2024-09-14T07:22:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:43.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:43.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_22.status 2024-09-14T07:22:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_22.status 2024-09-14T07:22:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:43.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2024-09-14T07:22:43.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:43.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:43.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386]]], state=connected] 2024-09-14T07:22:43.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=9388], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386], [object_number=9, tag_tid=1, entry_tid=9385]]], state=connected] 2024-09-14T07:22:43.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=9388' 2024-09-14T07:22:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=9388' 2024-09-14T07:22:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9388' '!=' '' 2024-09-14T07:22:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=9388' = '' 2024-09-14T07:22:43.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=1, entry_tid=9388' 2024-09-14T07:22:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:43.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:22:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=9388' 2024-09-14T07:22:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:44.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:44.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=9388' 2024-09-14T07:22:44.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:44.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:44.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9388 2024-09-14T07:22:44.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9388 2024-09-14T07:22:44.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:44.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:44.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_22 2024-09-14T07:22:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_22 ']' 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_22' 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:44.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:44.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:44.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:44.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:44.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:44.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:44.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:44.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2024-09-14T07:22:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T07:22:44.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:44.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:44.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:44.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2024-09-14T07:22:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-14T07:22:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:44.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:44.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:44.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:44.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:44.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:44.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2024-09-14T07:22:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_22 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_22 '' 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:44.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_22.status 2024-09-14T07:22:44.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_22.status 2024-09-14T07:22:44.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2024-09-14T07:22:44.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:44.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386]]], state=connected] 2024-09-14T07:22:44.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386]]], state=connected] 2024-09-14T07:22:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:44.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2024-09-14T07:22:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_22.status 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_22.status 2024-09-14T07:22:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2024-09-14T07:22:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:44.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:44.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:44.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:44.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386]]], state=connected] 2024-09-14T07:22:44.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9387], [object_number=10, tag_tid=1, entry_tid=9386]]], state=connected] 2024-09-14T07:22:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.status 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:22:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:45.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_22': 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0f5db56ba 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0f5db56ba 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:22:45.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:12:24 2024 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:12:24 2024 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:13:25 2024 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0f5db56ba 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: c4de7767-df91-475d-8d0a-56b6d6af2c47 2024-09-14T07:22:45.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_22 snap 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.export 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_22.export 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_22.export 2024-09-14T07:22:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.export 2024-09-14T07:22:45.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:45.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9408495021071469) > weight (0.25) 2024-09-14T07:22:45.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7996293923478083) > weight (0.25) 2024-09-14T07:22:45.998 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3949798572727393) > weight (0.25) 2024-09-14T07:22:45.999 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9614030473765273) > weight (0.25) 2024-09-14T07:22:45.999 INFO:tasks.rbd_mirror_thrash:waiting for 78 secs before thrashing 2024-09-14T07:22:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_22.export 2024-09-14T07:22:46.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:46.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_22.export 2024-09-14T07:22:46.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_22.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_22.export 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_23 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_23 snap 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:46.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:46.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:46.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:47.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:47.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2024-09-14T07:22:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:48.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:48.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:48.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:48.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:48.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:48.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:48.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:48.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:48.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:48.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:48.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:48.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:48.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:48.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2024-09-14T07:22:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T07:22:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2024-09-14T07:22:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:48.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:48.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:48.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2024-09-14T07:22:49.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:49.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_23 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_23 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:49.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:49.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:49.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_23 2024-09-14T07:22:49.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_23 ']' 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_23' 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:49.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:49.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:49.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:49.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:49.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:49.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:49.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:49.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:49.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:49.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:49.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:49.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:49.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:49.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T07:22:49.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:49.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:49.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:49.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:49.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:49.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:49.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:49.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:49.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_23 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '' 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:49.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:49.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:49.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:49.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:49.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:49.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:49.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:49.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2024-09-14T07:22:49.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:49.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:49.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:49.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:49.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:49.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:49.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:49.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:49.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:49.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9989' '!=' '' 2024-09-14T07:22:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:22:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9989' = '' 2024-09-14T07:22:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:22:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:22:49.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:49.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:22:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:22:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:22:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9989 2024-09-14T07:22:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9989 2024-09-14T07:22:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_23 2024-09-14T07:22:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_23 ']' 2024-09-14T07:22:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_23' 2024-09-14T07:22:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2024-09-14T07:22:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:49.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:49.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:49.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:49.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:49.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:49.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:49.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:49.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2024-09-14T07:22:49.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:49.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:49.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:49.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:49.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:49.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:49.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:49.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:49.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:49.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:49.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:49.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:49.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:49.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:49.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:49.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:49.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T07:22:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2024-09-14T07:22:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:50.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:50.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_23 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '' 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:50.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:50.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:50.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:50.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:50.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:50.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:50.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:50.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:50.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:50.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:50.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9989' '!=' 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:22:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_23 2024-09-14T07:22:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:50.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_23 ']' 2024-09-14T07:22:50.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_23' 2024-09-14T07:22:50.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2024-09-14T07:22:50.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:50.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:50.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:50.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:50.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:50.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2024-09-14T07:22:50.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:50.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:50.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:50.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:50.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:50.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:50.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:50.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T07:22:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:50.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:50.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:50.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:51.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_23 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '' 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:51.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:51.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:51.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:51.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:51.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:51.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:51.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:51.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:51.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:51.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:51.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:51.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:51.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:51.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:51.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:51.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:51.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:51.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:51.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:51.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9989' '!=' 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:51.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:51.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_23 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_23 ']' 2024-09-14T07:22:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_23' 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:51.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:52.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T07:22:52.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:52.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:52.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:52.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:52.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:52.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:52.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:52.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_23 2024-09-14T07:22:52.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '' 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:52.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:52.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:52.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:52.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:52.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:52.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:52.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:52.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:52.328 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:52.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:52.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=9989], [object_number=8, tag_tid=1, entry_tid=9988], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=9989' '!=' 'object_number=9, tag_tid=1, entry_tid=9989' 2024-09-14T07:22:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:52.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:22:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_23 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_23 ']' 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_23' 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:53.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:53.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:53.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:53.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:53.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:53.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:53.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:54.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:54.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:54.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:54.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:54.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-14T07:22:54.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:54.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:54.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:54.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:54.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:54.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:54.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:54.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:54.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:54.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:54.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:54.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:54.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:54.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:54.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2024-09-14T07:22:54.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:54.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_23 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '' 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:22:54.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:54.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:54.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:54.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2024-09-14T07:22:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_23.status 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_23.status 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2024-09-14T07:22:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:54.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:22:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:22:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:22:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=9987], [object_number=10, tag_tid=1, entry_tid=9986]]], state=connected] 2024-09-14T07:22:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.status 2024-09-14T07:22:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:22:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:55.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2024-09-14T07:22:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_23': 2024-09-14T07:22:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:22:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:22:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:22:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed41d2dad 2024-09-14T07:22:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed41d2dad 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:12:33 2024 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:12:33 2024 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:13:34 2024 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed41d2dad 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 67b9f81d-b60e-4557-9372-546d2732dac2 2024-09-14T07:22:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_23 snap 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.export 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_23.export 2024-09-14T07:22:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_23.export 2024-09-14T07:22:55.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.export 2024-09-14T07:22:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:55.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_23.export 2024-09-14T07:22:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:22:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_23.export 2024-09-14T07:22:56.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_23.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_23.export 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_24 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_24 snap 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:22:56.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:22:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2024-09-14T07:22:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:57.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:57.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:22:58.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2024-09-14T07:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:22:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:22:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2024-09-14T07:22:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:22:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:22:58.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:58.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:58.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:22:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2024-09-14T07:22:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:22:59.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:59.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:59.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:59.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2024-09-14T07:22:59.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:22:59.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:59.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_24 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:59.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:22:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:22:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:59.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_24 2024-09-14T07:22:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:22:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:22:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_24 2024-09-14T07:22:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:22:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:22:59.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:22:59.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:22:59.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:22:59.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:22:59.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:22:59.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:22:59.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:22:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:22:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:22:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_24 2024-09-14T07:22:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:22:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:22:59.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:22:59.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:22:59.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_24 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_24 2024-09-14T07:22:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:22:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_24 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_24 ']' 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_24' 2024-09-14T07:22:59.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:22:59.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:22:59.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:22:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2024-09-14T07:22:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:22:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:22:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:22:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:22:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:22:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:22:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:22:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:22:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2024-09-14T07:22:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:22:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:59.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:22:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:22:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:59.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:22:59.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:22:59.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:59.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:22:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:22:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:22:59.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:59.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:59.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:22:59.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2024-09-14T07:22:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:22:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:22:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_24 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:22:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:22:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:22:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:22:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:22:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:22:59.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:59.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:59.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:22:59.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:00.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:00.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:00.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_24 2024-09-14T07:23:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:23:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:23:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:23:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:23:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:23:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:23:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:23:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_24 2024-09-14T07:23:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '' 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:00.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:00.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:00.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:00.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:00.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:00.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:00.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:00.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10330], [object_number=9, tag_tid=1, entry_tid=10329], [object_number=8, tag_tid=1, entry_tid=10328], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:00.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2024-09-14T07:23:00.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:00.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:00.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:00.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:00.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:00.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:00.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:00.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:00.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:00.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:00.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:00.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10330], [object_number=9, tag_tid=1, entry_tid=10329], [object_number=8, tag_tid=1, entry_tid=10328], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:00.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10330' '!=' '' 2024-09-14T07:23:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:23:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10330' = '' 2024-09-14T07:23:00.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:00.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:00.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:00.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:23:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:00.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:00.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:23:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:00.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:23:00.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:00.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:00.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:00.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10330 2024-09-14T07:23:00.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 10330 2024-09-14T07:23:00.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:00.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:00.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_24 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_24 ']' 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_24' 2024-09-14T07:23:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:00.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:00.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:00.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:00.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2024-09-14T07:23:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:00.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:00.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_24 2024-09-14T07:23:00.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:00.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:00.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:00.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:00.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:00.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:00.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:00.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:00.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:00.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:00.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:00.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:00.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:00.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:00.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_24 2024-09-14T07:23:00.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:23:00.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:23:00.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:00.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:23:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:23:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:23:00.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:00.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:00.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:00.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:01.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_24 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '' 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:01.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:01.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:01.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:01.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:01.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:01.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:01.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:01.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:01.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10330], [object_number=9, tag_tid=1, entry_tid=10329], [object_number=8, tag_tid=1, entry_tid=10328], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:01.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:01.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:01.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2024-09-14T07:23:01.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:01.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:01.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:01.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:01.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:01.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:01.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:01.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:01.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:01.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10330], [object_number=9, tag_tid=1, entry_tid=10329], [object_number=8, tag_tid=1, entry_tid=10328], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:01.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:01.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:01.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10330' '!=' 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:01.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:01.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_24 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_24 ']' 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_24' 2024-09-14T07:23:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:01.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:01.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:01.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2024-09-14T07:23:01.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2024-09-14T07:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:01.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:01.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:01.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:01.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_24 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:01.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:01.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:01.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:01.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:01.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_24 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:23:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:23:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:23:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:23:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:23:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:23:02.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:02.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:02.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_24 2024-09-14T07:23:02.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:02.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:02.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:02.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '' 2024-09-14T07:23:02.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:02.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:02.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:02.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:02.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:02.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:02.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:02.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:02.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:02.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:02.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:02.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:02.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:02.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:02.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10330], [object_number=9, tag_tid=1, entry_tid=10329], [object_number=8, tag_tid=1, entry_tid=10328], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:02.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:02.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:02.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2024-09-14T07:23:02.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:02.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:02.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:02.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2024-09-14T07:23:02.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:02.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:02.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:02.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:02.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:02.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:02.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:02.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:02.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10330], [object_number=9, tag_tid=1, entry_tid=10329], [object_number=8, tag_tid=1, entry_tid=10328], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:02.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10330' '!=' 'object_number=10, tag_tid=1, entry_tid=10330' 2024-09-14T07:23:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_24 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_24 ']' 2024-09-14T07:23:03.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_24' 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:03.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:03.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:03.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:03.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:03.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:03.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:03.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:03.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:03.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2024-09-14T07:23:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:03.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:03.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:03.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:03.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:03.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:03.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:03.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_24 2024-09-14T07:23:03.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:03.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:03.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:03.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:03.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:03.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:03.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:03.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:03.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-14T07:23:03.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:03.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_24 2024-09-14T07:23:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:23:03.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:23:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:03.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:23:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:03.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:03.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:23:03.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:23:03.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:23:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:23:03.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_24 2024-09-14T07:23:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_24 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '' 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:03.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:03.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:03.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:03.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:03.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:03.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:03.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:03.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:03.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:03.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:03.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:03.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2024-09-14T07:23:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2024-09-14T07:23:03.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_24.status 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_24.status 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:03.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:03.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2024-09-14T07:23:03.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10330], [object_number=11, tag_tid=1, entry_tid=10327]]], state=connected] 2024-09-14T07:23:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.status 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:23:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:04.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2024-09-14T07:23:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_24': 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123b4b42a9fc 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123b4b42a9fc 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:12:45 2024 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:12:45 2024 2024-09-14T07:23:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:13:46 2024 2024-09-14T07:23:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:23:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123b4b42a9fc 2024-09-14T07:23:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:23:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:23:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: fcdaa752-813d-4c30-88c6-3649550c5e90 2024-09-14T07:23:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:23:04.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_24 snap 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.export 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_24.export 2024-09-14T07:23:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_24.export 2024-09-14T07:23:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.export 2024-09-14T07:23:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:05.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_24.export 2024-09-14T07:23:05.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:05.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_24.export 2024-09-14T07:23:05.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_24.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_24.export 2024-09-14T07:23:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:23:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_25 2024-09-14T07:23:05.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_25 snap 2024-09-14T07:23:05.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:05.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:05.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:05.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:05.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2024-09-14T07:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:06.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:07.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2024-09-14T07:23:07.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:07.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:07.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:23:07.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:07.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:07.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:23:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:07.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:07.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:07.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:07.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2024-09-14T07:23:08.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:23:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:23:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:23:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:23:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_25 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_25 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:08.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_25 2024-09-14T07:23:08.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:08.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_25 ']' 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_25' 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:08.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2024-09-14T07:23:08.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:08.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2024-09-14T07:23:08.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_25 2024-09-14T07:23:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '' 2024-09-14T07:23:08.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:08.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:08.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:08.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:08.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:08.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:08.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:08.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:08.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:08.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2024-09-14T07:23:08.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2024-09-14T07:23:08.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:08.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:08.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:08.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:08.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:08.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11614' '!=' '' 2024-09-14T07:23:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:23:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11614' = '' 2024-09-14T07:23:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:23:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:23:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:23:08.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:08.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11614 2024-09-14T07:23:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11614 2024-09-14T07:23:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:08.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:08.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_25 2024-09-14T07:23:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_25 ']' 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_25' 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:08.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:08.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2024-09-14T07:23:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2024-09-14T07:23:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_25 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '' 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:08.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:08.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:08.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:08.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:08.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:08.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:08.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:08.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:08.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:08.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:08.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:08.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:08.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:08.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:08.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11614' '!=' 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:08.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:08.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:23:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_25 2024-09-14T07:23:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_25 ']' 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_25' 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:09.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:09.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:09.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:09.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:09.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2024-09-14T07:23:09.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_25 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '' 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:09.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:09.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:09.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:09.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:09.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:09.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:09.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:09.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:09.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:09.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:09.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:09.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2024-09-14T07:23:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:09.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:09.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:09.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:09.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:09.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11614' '!=' 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:09.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:09.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:23:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_25 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_25 ']' 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_25' 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:10.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:10.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:10.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:10.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:10.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:10.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:10.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:10.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:10.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:10.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:10.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2024-09-14T07:23:10.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_25 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '' 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:10.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:10.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:10.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:10.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:10.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:10.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:10.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:10.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:10.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:10.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:10.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:10.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:10.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:10.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11614' '!=' 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:10.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:10.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:23:12.005 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5906142718787234) > weight (0.25) 2024-09-14T07:23:12.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.507159156442197) > weight (0.25) 2024-09-14T07:23:12.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7980739334598216) > weight (0.25) 2024-09-14T07:23:12.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6459991179761625) > weight (0.25) 2024-09-14T07:23:12.006 INFO:tasks.rbd_mirror_thrash:waiting for 89 secs before thrashing 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_25 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_25 ']' 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_25' 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2024-09-14T07:23:12.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2024-09-14T07:23:12.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:12.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_25 2024-09-14T07:23:12.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:12.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:12.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:12.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '' 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:12.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:12.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:12.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:12.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:12.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:12.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:12.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:12.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:12.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11614], [object_number=9, tag_tid=1, entry_tid=11613], [object_number=8, tag_tid=1, entry_tid=11612], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:12.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11614' '!=' 'object_number=10, tag_tid=1, entry_tid=11614' 2024-09-14T07:23:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:23:14.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_25 2024-09-14T07:23:14.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:14.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:14.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:14.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_25 ']' 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_25' 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:14.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:14.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:14.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:14.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:14.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:14.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2024-09-14T07:23:14.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:15.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_25 2024-09-14T07:23:15.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '' 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:15.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:15.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:15.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:15.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:15.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:15.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:15.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:23:15.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:15.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_25.status 2024-09-14T07:23:15.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_25.status 2024-09-14T07:23:15.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:15.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:15.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2024-09-14T07:23:15.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:15.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:15.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:15.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:15.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:15.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11614], [object_number=11, tag_tid=1, entry_tid=11611]]], state=connected] 2024-09-14T07:23:15.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.status 2024-09-14T07:23:15.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:23:15.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:16.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2024-09-14T07:23:16.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_25': 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11829073f075 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11829073f075 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:12:59 2024 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:12:59 2024 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:00 2024 2024-09-14T07:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11829073f075 2024-09-14T07:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 4b109641-f23e-48e0-9065-a1f448eaba74 2024-09-14T07:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_25 snap 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.export 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_25.export 2024-09-14T07:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_25.export 2024-09-14T07:23:16.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.export 2024-09-14T07:23:16.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:16.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_25.export 2024-09-14T07:23:16.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:16.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_25.export 2024-09-14T07:23:16.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_25.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_25.export 2024-09-14T07:23:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:23:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_26 2024-09-14T07:23:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_26 snap 2024-09-14T07:23:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:16.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:16.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:16.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:16.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:23:17.867 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.861+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca54f27a00 2024-09-14T07:23:17.868 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.861+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca59a8a340 2024-09-14T07:23:17.868 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.861+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca6ba68dd0 2024-09-14T07:23:17.868 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.861+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca72e09110 2024-09-14T07:23:17.868 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.865+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca73050820 2024-09-14T07:23:17.868 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.865+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca86633e10 2024-09-14T07:23:17.868 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:23:17.865+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca86c56f70 2024-09-14T07:23:19.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:19.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2024-09-14T07:23:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:20.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:20.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:23:20.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:23:20.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:23:20.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:20.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:20.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:23:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2024-09-14T07:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:23:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:23:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_26 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_26 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:21.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_26 2024-09-14T07:23:21.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:21.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:21.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_26 ']' 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_26' 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:21.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:21.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:21.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:21.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:21.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:21.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:21.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:21.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2024-09-14T07:23:21.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_26 2024-09-14T07:23:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '' 2024-09-14T07:23:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:21.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:21.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:21.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:21.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:21.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:21.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:21.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:21.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:21.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:21.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6023' '!=' '' 2024-09-14T07:23:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:23:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6023' = '' 2024-09-14T07:23:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:23:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:21.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:23:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:21.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:21.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:23:21.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:21.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:21.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=6023 2024-09-14T07:23:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 6023 2024-09-14T07:23:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:21.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_26 2024-09-14T07:23:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:21.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_26 ']' 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_26' 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:21.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:21.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2024-09-14T07:23:21.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_26 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '' 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:21.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:21.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:21.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:21.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:21.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:21.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:21.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:21.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:21.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:21.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:21.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:21.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6023' '!=' 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:23:22.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_26 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_26 ']' 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_26' 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:22.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:22.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:22.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:22.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:22.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:22.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:22.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2024-09-14T07:23:22.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:22.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_26 2024-09-14T07:23:22.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:22.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:22.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:22.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '' 2024-09-14T07:23:22.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:22.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:22.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:22.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:22.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:22.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:22.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:22.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:22.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:22.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:22.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:22.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:22.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:22.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:22.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:22.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:22.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:22.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:22.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:22.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:22.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:22.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6023' '!=' 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:22.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:22.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:23:23.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_26 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_26 ']' 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_26' 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:23.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:23.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:23.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:23.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:23.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2024-09-14T07:23:23.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:23.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:23.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2024-09-14T07:23:23.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_26 2024-09-14T07:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '' 2024-09-14T07:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:23.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:23.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:23.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:23.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:23.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:23.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:23.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:23.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:23.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:23.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:23.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:23.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:23.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022], [object_number=5, tag_tid=1, entry_tid=6021], [object_number=4, tag_tid=1, entry_tid=6020]]], state=connected] 2024-09-14T07:23:23.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:23.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:23.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:23.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6023' '!=' 'object_number=7, tag_tid=1, entry_tid=6023' 2024-09-14T07:23:23.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:23.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_26 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_26 ']' 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_26' 2024-09-14T07:23:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:25.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:25.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:25.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:25.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:25.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2024-09-14T07:23:25.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:25.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_26 2024-09-14T07:23:25.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:25.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:25.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '' 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:25.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:25.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:25.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:25.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:25.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:25.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2024-09-14T07:23:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_26.status 2024-09-14T07:23:25.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_26.status 2024-09-14T07:23:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2024-09-14T07:23:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:25.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:25.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6023], [object_number=6, tag_tid=1, entry_tid=6022]]], state=connected] 2024-09-14T07:23:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.status 2024-09-14T07:23:25.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:25.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:25.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:25.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:23:25.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:26.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2024-09-14T07:23:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_26': 2024-09-14T07:23:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:23:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:23:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:23:26.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0f5593327 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0f5593327 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:13:10 2024 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:13:10 2024 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:13:10 2024 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0f5593327 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a5f386b1-2078-41c0-b309-832d95eedd4c 2024-09-14T07:23:26.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_26 snap 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.export 2024-09-14T07:23:26.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_26.export 2024-09-14T07:23:26.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_26.export 2024-09-14T07:23:26.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.export 2024-09-14T07:23:26.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:26.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_26.export 2024-09-14T07:23:27.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_26.export 2024-09-14T07:23:27.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_26.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_26.export 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_27 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_27 snap 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:27.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:23:28.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:29.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2024-09-14T07:23:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:23:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:23:29.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:23:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:23:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:23:29.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:29.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:29.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:29.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:29.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2024-09-14T07:23:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:23:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:29.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:29.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:29.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:29.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:29.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:23:29.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2024-09-14T07:23:30.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:30.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:23:30.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_27 2024-09-14T07:23:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:23:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:30.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:23:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:23:30.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_27 2024-09-14T07:23:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:23:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:23:30.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:23:30.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:23:30.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:23:30.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_27 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_27 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:30.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_27 2024-09-14T07:23:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_27 ']' 2024-09-14T07:23:30.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_27' 2024-09-14T07:23:30.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2024-09-14T07:23:30.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:30.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:30.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:30.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:30.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:30.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:30.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:30.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:30.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:30.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:30.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:30.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2024-09-14T07:23:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:30.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:30.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:30.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:30.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:30.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:30.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:30.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_27 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '' 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:30.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:30.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:30.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:30.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610], [object_number=5, tag_tid=1, entry_tid=6609], [object_number=4, tag_tid=1, entry_tid=6608]]], state=connected] 2024-09-14T07:23:30.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:30.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:30.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:30.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610], [object_number=5, tag_tid=1, entry_tid=6609], [object_number=4, tag_tid=1, entry_tid=6608]]], state=connected] 2024-09-14T07:23:30.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:30.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:30.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:30.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6611' '!=' '' 2024-09-14T07:23:30.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:23:30.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6611' = '' 2024-09-14T07:23:30.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:30.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:30.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:30.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:30.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:23:30.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:30.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:30.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:23:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:30.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:23:30.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:30.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:30.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=6611 2024-09-14T07:23:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 6611 2024-09-14T07:23:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:31.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_27 2024-09-14T07:23:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_27 ']' 2024-09-14T07:23:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_27' 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:31.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:31.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:31.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:31.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:31.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:31.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:31.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:31.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:31.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:31.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:31.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:31.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:31.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:31.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2024-09-14T07:23:31.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:31.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:31.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:31.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:31.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:31.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:31.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:31.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:31.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:31.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:31.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:31.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:31.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:31.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:31.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:31.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:31.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_27 2024-09-14T07:23:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '' 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:31.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:31.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:31.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610], [object_number=5, tag_tid=1, entry_tid=6609], [object_number=4, tag_tid=1, entry_tid=6608]]], state=connected] 2024-09-14T07:23:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2024-09-14T07:23:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:31.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:31.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:31.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:31.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:31.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:31.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610], [object_number=5, tag_tid=1, entry_tid=6609], [object_number=4, tag_tid=1, entry_tid=6608]]], state=connected] 2024-09-14T07:23:31.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:31.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:31.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:31.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6611' '!=' 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:31.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:31.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:23:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_27 2024-09-14T07:23:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_27 ']' 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_27' 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:32.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:32.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:32.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:32.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:32.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:32.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:32.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:32.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:32.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:32.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:32.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:32.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:32.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:32.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:32.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:32.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:32.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:32.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:32.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:32.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2024-09-14T07:23:32.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:32.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:32.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:32.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:32.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_27 2024-09-14T07:23:32.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:32.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '' 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:32.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:32.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:32.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:32.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:32.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:32.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:32.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:32.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:32.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:32.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:32.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610], [object_number=5, tag_tid=1, entry_tid=6609], [object_number=4, tag_tid=1, entry_tid=6608]]], state=connected] 2024-09-14T07:23:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:32.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:32.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:32.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:32.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610], [object_number=5, tag_tid=1, entry_tid=6609], [object_number=4, tag_tid=1, entry_tid=6608]]], state=connected] 2024-09-14T07:23:32.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:32.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:32.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:32.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=6611' '!=' 'object_number=7, tag_tid=1, entry_tid=6611' 2024-09-14T07:23:32.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:32.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:23:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_27 2024-09-14T07:23:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_27 ']' 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_27' 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:33.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2024-09-14T07:23:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:33.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:33.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:33.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:33.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2024-09-14T07:23:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:33.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:33.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:33.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:33.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:33.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:33.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-14T07:23:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_27 2024-09-14T07:23:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:23:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:23:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:23:33.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:23:33.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:33.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:33.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:23:33.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:23:33.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:23:33.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:33.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:33.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:23:33.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_27 2024-09-14T07:23:33.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:33.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_27 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '' 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:33.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:33.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:33.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:33.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:33.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:33.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:33.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:33.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:33.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:33.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:33.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:33.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:33.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:33.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2024-09-14T07:23:33.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_27.status 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_27.status 2024-09-14T07:23:33.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2024-09-14T07:23:33.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:34.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:23:34.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:23:34.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:34.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:34.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:23:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.status 2024-09-14T07:23:34.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:34.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:23:34.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:34.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2024-09-14T07:23:34.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:34.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:34.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:34.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:34.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:34.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:23:34.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:35.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2024-09-14T07:23:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_27': 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed669a27c7 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed669a27c7 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:23:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:13:19 2024 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:13:19 2024 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:13:19 2024 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed669a27c7 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 3de84141-3470-4a02-b51f-7a6be8836da1 2024-09-14T07:23:35.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:23:35.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:35.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_27 snap 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.export 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_27.export 2024-09-14T07:23:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_27.export 2024-09-14T07:23:35.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.export 2024-09-14T07:23:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:35.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_27.export 2024-09-14T07:23:35.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_27.export 2024-09-14T07:23:35.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_27.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_27.export 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_28 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_28 snap 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:35.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:38.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:38.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2024-09-14T07:23:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:39.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:39.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:39.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:23:39.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:23:39.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:39.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:23:39.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:39.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:23:39.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:39.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2024-09-14T07:23:39.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:23:39.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:39.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:23:39.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:23:39.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:39.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2024-09-14T07:23:39.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T07:23:39.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:23:39.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:23:39.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:23:39.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:39.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2024-09-14T07:23:40.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:23:40.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:23:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:23:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:23:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_28 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:23:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_28 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_28 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_28 ']' 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_28' 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:40.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:40.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:40.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:40.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:40.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:40.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:40.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:40.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:40.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2024-09-14T07:23:40.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:40.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:40.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:40.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:40.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:40.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:40.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:40.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:40.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T07:23:40.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:40.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:40.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:40.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:40.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:40.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_28 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '' 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:40.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:40.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:40.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:40.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:40.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:40.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:40.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2024-09-14T07:23:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2024-09-14T07:23:40.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:40.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:40.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:40.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11044' '!=' '' 2024-09-14T07:23:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:23:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11044' = '' 2024-09-14T07:23:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:40.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:23:40.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:40.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:40.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:40.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:23:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:40.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:23:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11044 2024-09-14T07:23:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11044 2024-09-14T07:23:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:40.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_28 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_28 ']' 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_28' 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:40.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:40.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2024-09-14T07:23:40.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:40.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:40.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:40.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:40.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:40.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:40.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:40.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:41.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T07:23:41.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:41.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:41.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_28 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '' 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:41.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:41.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:41.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:41.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:41.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:41.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:41.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:41.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:41.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:41.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:41.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2024-09-14T07:23:41.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:41.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:41.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:41.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:41.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:41.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:41.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:41.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:41.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:41.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:41.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11044' '!=' 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_28 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_28 ']' 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_28' 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:41.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:41.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:41.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:41.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:41.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:41.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:41.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:41.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:41.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:41.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T07:23:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2024-09-14T07:23:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:41.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:41.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:41.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:41.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:41.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:41.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:41.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_28 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '' 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:42.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:42.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:42.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:42.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:42.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:42.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:42.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:42.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:42.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:42.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:42.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:42.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:42.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:42.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:42.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:42.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:42.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:42.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11044' '!=' 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:42.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:42.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:23:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_28 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_28 ']' 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_28' 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2024-09-14T07:23:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:42.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T07:23:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:43.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:43.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:43.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:43.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:43.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:43.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_28 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '' 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:43.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:43.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:43.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:43.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:43.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:43.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:43.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:43.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:43.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2024-09-14T07:23:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:43.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:43.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:43.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:43.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:43.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:43.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=11044], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042], [object_number=9, tag_tid=1, entry_tid=11041]]], state=connected] 2024-09-14T07:23:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=11044' '!=' 'object_number=8, tag_tid=1, entry_tid=11044' 2024-09-14T07:23:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_28 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_28 ']' 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_28' 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:45.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:45.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:45.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:45.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:45.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:45.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:45.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:45.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-14T07:23:45.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:23:45.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:23:45.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:23:45.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:23:45.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:45.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:45.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:23:45.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:23:45.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:23:45.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:23:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2024-09-14T07:23:45.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:45.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_28 2024-09-14T07:23:45.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '' 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:45.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:45.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:45.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:45.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:45.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:45.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:45.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:45.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:45.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2024-09-14T07:23:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2024-09-14T07:23:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:23:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_28.status 2024-09-14T07:23:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_28.status 2024-09-14T07:23:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:45.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:45.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2024-09-14T07:23:45.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11043], [object_number=10, tag_tid=1, entry_tid=11042]]], state=connected] 2024-09-14T07:23:45.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.status 2024-09-14T07:23:45.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:23:45.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:46.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2024-09-14T07:23:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_28': 2024-09-14T07:23:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:23:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:23:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:23:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 123bb3676076 2024-09-14T07:23:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.123bb3676076 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:13:28 2024 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:13:28 2024 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:42 2024 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 123bb3676076 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 9f1fe7ea-871e-472c-a3b1-7f76d0a17755 2024-09-14T07:23:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:23:46.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_28 snap 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.export 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_28.export 2024-09-14T07:23:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_28.export 2024-09-14T07:23:46.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.export 2024-09-14T07:23:46.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_28.export 2024-09-14T07:23:46.972 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:23:46.971+0000 7f99e021c640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:23:46.974 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:23:46.971+0000 7f99dca15640 -1 bdev(0x55920ea9cc00 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2024-09-14T07:23:47.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:47.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_28.export 2024-09-14T07:23:47.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_28.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_28.export 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_29 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_29 snap 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:48.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:49.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2024-09-14T07:23:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:23:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:23:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:23:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2024-09-14T07:23:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:23:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:23:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:23:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:23:49.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:23:49.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:23:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:23:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:23:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:23:49.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2024-09-14T07:23:49.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:23:49.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:23:49.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:23:49.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:23:49.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_29 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:49.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_29 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_29 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_29 ']' 2024-09-14T07:23:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_29' 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:50.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:50.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:50.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:50.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:50.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:50.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:50.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:50.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2024-09-14T07:23:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_29 2024-09-14T07:23:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '' 2024-09-14T07:23:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:50.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:50.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:50.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:50.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:50.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:50.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:50.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10826' '!=' '' 2024-09-14T07:23:50.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:23:50.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10826' = '' 2024-09-14T07:23:50.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:23:50.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:23:50.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:50.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:23:50.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:50.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:50.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:23:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:23:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:23:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10826 2024-09-14T07:23:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 10826 2024-09-14T07:23:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:23:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_29 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_29 ']' 2024-09-14T07:23:50.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_29' 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:50.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:50.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:50.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:50.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:50.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:50.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:50.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:50.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:50.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:50.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:50.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2024-09-14T07:23:50.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_29 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '' 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:50.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:50.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:50.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:50.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:50.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:50.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2024-09-14T07:23:50.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:50.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:50.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:50.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10826' '!=' 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:50.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:50.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_29 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_29 ']' 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_29' 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2024-09-14T07:23:51.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:51.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:51.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:51.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2024-09-14T07:23:51.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:51.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:51.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:51.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:51.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2024-09-14T07:23:51.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:51.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_29 2024-09-14T07:23:51.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '' 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:51.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:51.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:51.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:51.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:51.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2024-09-14T07:23:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:51.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:51.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:51.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:51.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:51.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:51.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:51.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:51.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:51.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10826' '!=' 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:51.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:51.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:23:52.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_29 2024-09-14T07:23:52.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:52.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_29 ']' 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_29' 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2024-09-14T07:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:52.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:52.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:52.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:52.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:52.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:52.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:52.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:52.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:52.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:52.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:52.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2024-09-14T07:23:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_29 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '' 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2024-09-14T07:23:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:52.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:52.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:52.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:52.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:52.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:52.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:52.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:52.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:52.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:52.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:52.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:52.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10826' '!=' 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:52.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:52.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:23:54.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_29 2024-09-14T07:23:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_29 ']' 2024-09-14T07:23:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_29' 2024-09-14T07:23:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2024-09-14T07:23:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:54.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:54.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:54.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:54.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:54.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:54.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:54.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:54.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:54.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:54.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:54.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:54.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2024-09-14T07:23:54.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_29 2024-09-14T07:23:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '' 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:54.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:54.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:54.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:54.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:54.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:54.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:54.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:54.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:54.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:54.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:54.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:54.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:54.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:54.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:54.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:54.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=10826], [object_number=9, tag_tid=1, entry_tid=10825], [object_number=8, tag_tid=1, entry_tid=10824], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:54.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:54.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:54.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:54.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=10826' '!=' 'object_number=10, tag_tid=1, entry_tid=10826' 2024-09-14T07:23:54.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:23:54.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:23:56.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_29 2024-09-14T07:23:56.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:56.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:56.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_29 ']' 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_29' 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:23:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:23:56.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:23:56.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:23:56.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:23:56.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:23:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:23:56.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:56.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:23:56.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2024-09-14T07:23:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:56.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_29 2024-09-14T07:23:56.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:56.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '' 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:56.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:56.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:56.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2024-09-14T07:23:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_29.status 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_29.status 2024-09-14T07:23:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:23:56.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:56.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2024-09-14T07:23:56.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:56.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:23:56.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:23:56.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:23:56.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:56.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=10826], [object_number=11, tag_tid=1, entry_tid=10823]]], state=connected] 2024-09-14T07:23:56.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.status 2024-09-14T07:23:56.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:23:56.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:23:57.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2024-09-14T07:23:57.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_29': 2024-09-14T07:23:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:23:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:23:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:23:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11825b01c1e5 2024-09-14T07:23:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11825b01c1e5 2024-09-14T07:23:57.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:23:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:23:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:23:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:23:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:13:41 2024 2024-09-14T07:23:57.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:13:41 2024 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:42 2024 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11825b01c1e5 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 16dde50d-5378-4a0d-a8f7-64fea09eb0e9 2024-09-14T07:23:57.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_29 snap 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.export 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_29.export 2024-09-14T07:23:57.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_29.export 2024-09-14T07:23:57.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.export 2024-09-14T07:23:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:58.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_29.export 2024-09-14T07:23:58.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:23:58.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_29.export 2024-09-14T07:23:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_29.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_29.export 2024-09-14T07:23:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:23:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:23:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_30 2024-09-14T07:23:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_30 snap 2024-09-14T07:23:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:23:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:23:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:23:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:23:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:24:01.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2024-09-14T07:24:01.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:01.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:01.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:01.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:24:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2024-09-14T07:24:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:24:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:24:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:24:02.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2024-09-14T07:24:02.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:24:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:24:02.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:02.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2024-09-14T07:24:02.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:24:02.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:24:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:02.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:02.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:02.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:02.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2024-09-14T07:24:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:24:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:24:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:24:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:24:03.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:24:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:24:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:24:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_30 2024-09-14T07:24:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:24:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_30 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:24:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:24:03.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:03.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_30 2024-09-14T07:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:03.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_30 ']' 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_30' 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:03.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:03.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:03.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:03.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:03.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:03.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:03.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:03.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:03.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:03.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:03.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:03.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:03.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_30 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '' 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:03.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:03.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:03.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:03.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:03.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:03.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2024-09-14T07:24:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:03.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:03.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10617' '!=' '' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10617' = '' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:24:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:24:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:03.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:24:03.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:03.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:24:03.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:03.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:24:03.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:03.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:24:03.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:03.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:24:03.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10617 2024-09-14T07:24:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 10617 2024-09-14T07:24:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:24:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:24:03.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_30 2024-09-14T07:24:03.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:03.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:03.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_30 ']' 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_30' 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:03.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:03.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:03.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:03.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:03.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:03.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:03.999 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:24:04.000 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:24:04.000 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T07:24:04.001 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:24:04.000+0000 7f9388619640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 96181) UID: 1000 2024-09-14T07:24:04.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:04.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_30 2024-09-14T07:24:04.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:04.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:04.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:04.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '' 2024-09-14T07:24:04.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:04.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:04.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:04.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:04.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:04.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:04.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:04.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:04.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:04.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:04.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:04.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:04.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:04.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:04.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:04.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:04.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:04.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:04.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:04.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10617' '!=' 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:24:04.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_30 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_30 ']' 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_30' 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:04.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:04.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:04.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:04.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:04.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:04.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:04.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:04.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:04.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:04.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:04.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:04.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:04.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:04.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:04.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:04.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:04.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:04.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:24:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:04.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:04.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:04.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:04.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:04.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:04.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:04.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:04.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:04.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:04.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:04.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:04.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:04.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:05.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:05.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:05.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:05.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:05.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:05.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:05.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:05.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:05.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:05.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:05.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:05.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:24:06.000 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:24:06.001 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:24:06.001 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:24:06.001 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:24:06.001 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:24:06.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:24:06.005 INFO:tasks.rbd_mirror_thrash:waiting for 74 secs before thrashing 2024-09-14T07:24:06.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:06.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:06.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:06.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:06.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:06.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:06.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:06.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:06.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:06.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:06.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:06.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:06.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:06.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:06.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:06.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:06.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:06.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:06.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:06.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:06.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:06.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:06.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:06.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:06.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:06.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:24:08.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:08.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:08.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:08.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:08.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:08.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:08.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:08.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:08.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:08.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:08.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:08.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:08.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:08.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:08.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:08.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:08.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:08.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:08.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:08.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:08.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:08.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:08.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:08.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:08.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:08.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:08.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:08.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:08.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:08.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:08.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:08.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:08.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:09.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:09.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:09.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:09.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:09.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:09.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:09.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:09.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:09.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:09.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:09.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:09.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:09.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:09.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:09.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:09.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:09.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:09.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:09.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:09.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:09.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 4 2024-09-14T07:24:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:13.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:13.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:13.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:13.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:13.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:13.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:13.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:13.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:13.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:13.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:13.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:13.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:13.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:13.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:13.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:13.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:13.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:13.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:13.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:13.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:13.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:13.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:13.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:13.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:13.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:13.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:13.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:13.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:13.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:13.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:24:21.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:21.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:21.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:21.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:21.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:21.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:21.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:21.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:21.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:21.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:21.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:21.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:21.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:22.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:22.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:22.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:22.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:22.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:22.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:22.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:22.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:22.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:22.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:22.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:22.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:22.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:22.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:22.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:22.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:22.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:22.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:22.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:22.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:22.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:24:30.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:30.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:30.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:30.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:30.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:30.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:30.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:30.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:30.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:30.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:30.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:30.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:30.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:30.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:30.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:30.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:30.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:30.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:30.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:30.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:30.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:30.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:30.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:30.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:30.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:30.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:30.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:30.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:30.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:30.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:30.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:30.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:30.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 8 2024-09-14T07:24:33.549 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:24:33.539+0000 7fb3e9154640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 3 2024-09-14T07:24:33.549 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:24:33.543+0000 7fb3e614e640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:24:33.597 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:24:33.595+0000 7f7dd8465640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 2 2024-09-14T07:24:33.662 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:24:33.663+0000 7f7c255b6640 -1 bdev(0x55ba904d4c00 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 3 2024-09-14T07:24:33.757 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:24:33.755+0000 7f7dda469640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 2 2024-09-14T07:24:33.758 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:24:33.759+0000 7f7dd8465640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 3 2024-09-14T07:24:38.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:38.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:38.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:38.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:38.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:38.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:38.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:38.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:38.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:38.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:39.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:39.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:39.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:39.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:39.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:39.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:39.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:39.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:39.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:39.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:39.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:39.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:39.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:39.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:39.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:39.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:39.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:39.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:39.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:39.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:39.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:39.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:39.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:39.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_30 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '' 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:39.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:39.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:39.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:39.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:39.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:39.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:39.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:39.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:39.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:39.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:39.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2024-09-14T07:24:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2024-09-14T07:24:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:39.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:39.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:39.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:39.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:39.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:39.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:39.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10617' '!=' 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:24:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_30 2024-09-14T07:24:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_30 ']' 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_30' 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:40.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:40.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:40.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:40.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:40.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:40.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:40.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:40.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:40.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:40.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:40.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:40.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:40.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:40.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:40.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:40.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:40.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:40.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:40.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:40.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:40.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:40.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:40.886 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:40.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:40.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:40.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:40.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:40.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:40.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:40.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:40.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:40.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:41.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.39738222976344495) > weight (0.25) 2024-09-14T07:24:41.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.801138550371325) > weight (0.25) 2024-09-14T07:24:41.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6928448202809968) > weight (0.25) 2024-09-14T07:24:41.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5248509595346235) > weight (0.25) 2024-09-14T07:24:41.008 INFO:tasks.rbd_mirror_thrash:waiting for 103 secs before thrashing 2024-09-14T07:24:41.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:41.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_30 2024-09-14T07:24:41.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:41.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '' 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:41.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:41.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:41.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:41.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:41.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:41.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:41.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:41.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:41.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2024-09-14T07:24:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2024-09-14T07:24:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:41.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:41.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10617' '!=' 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:24:42.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_30 2024-09-14T07:24:42.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:42.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:42.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:42.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_30 ']' 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_30' 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:42.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:42.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:42.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:42.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:42.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:42.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:42.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:42.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:42.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:42.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:42.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:42.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:42.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:42.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:42.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:43.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:43.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:43.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:43.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:43.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:43.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:43.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:43.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:43.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_30 2024-09-14T07:24:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '' 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:43.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:43.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:43.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2024-09-14T07:24:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:43.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=1, entry_tid=10617], [object_number=8, tag_tid=1, entry_tid=10616], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:43.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:43.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:43.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:43.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=9, tag_tid=1, entry_tid=10617' '!=' 'object_number=9, tag_tid=1, entry_tid=10617' 2024-09-14T07:24:43.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:43.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:24:45.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_30 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_30 ']' 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_30' 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:45.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:45.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:45.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:45.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:45.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:45.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:45.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:45.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:45.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:45.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:24:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:24:45.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:24:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:24:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:24:45.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:24:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:24:45.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:45.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:45.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:24:45.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:45.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:45.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:24:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:24:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:24:45.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:45.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:45.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:24:45.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-14T07:24:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:24:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:24:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2024-09-14T07:24:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:24:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:45.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:45.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:24:45.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:45.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:24:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:24:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:24:45.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:45.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:24:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2024-09-14T07:24:46.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:46.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_30 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '' 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:46.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:46.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:46.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:46.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:46.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:46.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:46.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:46.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:46.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:24:46.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_30.status 2024-09-14T07:24:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_30.status 2024-09-14T07:24:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:46.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2024-09-14T07:24:46.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:46.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:46.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:46.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:46.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:46.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10615], [object_number=10, tag_tid=1, entry_tid=10614]]], state=connected] 2024-09-14T07:24:46.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.status 2024-09-14T07:24:46.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:46.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:46.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:46.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2024-09-14T07:24:46.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:46.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:46.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:46.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:24:46.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:24:46.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:24:46.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:24:47.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_30': 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0e1618548 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0e1618548 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:13:53 2024 2024-09-14T07:24:47.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:13:53 2024 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:54 2024 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0e1618548 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a04b02dc-9337-4f48-9232-5fb95d990ec8 2024-09-14T07:24:47.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:24:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_30 snap 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.export 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_30.export 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_30.export 2024-09-14T07:24:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.export 2024-09-14T07:24:47.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:24:47.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_30.export 2024-09-14T07:24:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:24:47.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_30.export 2024-09-14T07:24:47.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_30.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_30.export 2024-09-14T07:24:47.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:47.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:24:47.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_31 2024-09-14T07:24:47.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_31 snap 2024-09-14T07:24:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:24:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:24:47.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:24:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:24:49.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:49.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:24:50.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2024-09-14T07:24:50.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:50.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:24:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:24:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:24:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2024-09-14T07:24:50.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:50.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:24:50.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:50.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:24:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2024-09-14T07:24:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:24:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:24:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:50.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:50.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:24:50.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:24:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:50.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2024-09-14T07:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:24:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:24:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:24:50.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:24:50.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_31 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:24:50.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_31 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:24:51.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:51.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:51.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:51.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:51.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:51.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:51.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:51.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:24:51.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:51.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:51.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:51.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:51.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:51.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:51.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' '!=' '' 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' = '' 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:51.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:24:51.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:51.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:24:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:24:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:51.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:24:51.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:51.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:24:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:24:51.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:24:51.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:24:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11786 2024-09-14T07:24:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11786 2024-09-14T07:24:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:24:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:24:51.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:51.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:51.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:51.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:51.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:51.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:24:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:51.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:51.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:51.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:51.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:51.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:51.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:51.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' '!=' 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:51.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:51.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:24:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:52.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:52.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:52.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:52.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:52.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:52.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:52.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:52.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:52.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:52.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:52.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:24:52.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:52.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:24:52.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:52.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:52.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:52.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:52.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:52.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:52.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:52.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:52.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:52.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:52.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:52.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' '!=' 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:53.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:53.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:53.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:53.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:53.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:53.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:53.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:24:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:53.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:53.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:53.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:53.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:53.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:53.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:53.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:53.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:53.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:53.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:24:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:53.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:53.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:53.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:53.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:53.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:53.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:53.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:53.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:53.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:53.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:53.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' '!=' 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:53.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:53.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:24:55.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:55.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:55.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:55.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:55.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:55.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:55.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:55.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:55.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:55.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:55.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:55.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:55.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:24:55.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:55.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:55.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:55.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:55.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:55.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:55.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:55.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:55.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:55.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:55.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:55.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:24:55.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:55.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:55.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:55.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:55.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:55.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:55.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' '!=' 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:55.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:24:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:57.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:57.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:57.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:57.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:57.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:57.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:57.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:57.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:57.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:57.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:57.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:57.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:57.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:57.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:24:57.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:24:57.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:24:57.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:57.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:57.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:57.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:24:57.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:57.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:57.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:57.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:24:57.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:57.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:57.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:57.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:57.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:57.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:57.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:24:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:24:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:24:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:24:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:24:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:24:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:24:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:57.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:24:57.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:24:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:24:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:24:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:57.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11786], [object_number=9, tag_tid=1, entry_tid=11785], [object_number=8, tag_tid=1, entry_tid=11784], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:24:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:24:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11786' '!=' 'object_number=10, tag_tid=1, entry_tid=11786' 2024-09-14T07:24:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:24:57.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:24:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_31 2024-09-14T07:24:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:24:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_31 ']' 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_31' 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:24:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:24:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:24:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:24:59.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:24:59.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:24:59.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:24:59.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:24:59.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2024-09-14T07:24:59.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:24:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:24:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:24:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:24:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:24:59.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:24:59.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:24:59.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:59.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:59.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:24:59.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2024-09-14T07:25:00.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_31 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '' 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:25:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:25:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:25:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:00.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:25:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:25:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:25:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:25:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:25:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:25:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:00.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2024-09-14T07:25:00.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_31.status 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_31.status 2024-09-14T07:25:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:25:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2024-09-14T07:25:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:25:00.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:00.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:00.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:00.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:25:00.112 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11786], [object_number=11, tag_tid=1, entry_tid=11783]]], state=connected] 2024-09-14T07:25:00.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.status 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:00.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:25:00.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:00.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:00.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:00.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:01.118 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2024-09-14T07:25:01.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_31': 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed5d84eb26 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed5d84eb26 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:14:06 2024 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:14:06 2024 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:15:07 2024 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed5d84eb26 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 0f03b0d6-d59a-4219-b7bc-f5c7f8ac0d93 2024-09-14T07:25:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_31 snap 2024-09-14T07:25:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:25:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:01.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:01.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.export 2024-09-14T07:25:01.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_31.export 2024-09-14T07:25:01.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_31.export 2024-09-14T07:25:01.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.export 2024-09-14T07:25:01.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_31.export 2024-09-14T07:25:01.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:01.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_31.export 2024-09-14T07:25:01.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_31.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_31.export 2024-09-14T07:25:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_32 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_32 snap 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:02.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:02.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:03.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2024-09-14T07:25:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:03.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:03.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:03.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:03.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2024-09-14T07:25:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:03.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:03.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:03.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:03.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:03.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:03.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:03.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:03.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2024-09-14T07:25:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2024-09-14T07:25:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:03.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:25:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:25:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:03.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:03.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:03.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:03.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2024-09-14T07:25:04.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:25:04.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:25:04.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:25:04.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:25:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:25:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_32 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_32 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:04.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_32 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_32 ']' 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_32' 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:04.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:04.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2024-09-14T07:25:04.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2024-09-14T07:25:04.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:04.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:04.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:04.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:04.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:04.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:04.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:04.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2024-09-14T07:25:04.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_32 2024-09-14T07:25:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_32 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_32 '' 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_32 2024-09-14T07:25:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_32.status 2024-09-14T07:25:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_32.status 2024-09-14T07:25:04.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:04.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.status 2024-09-14T07:25:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2024-09-14T07:25:04.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.status 2024-09-14T07:25:04.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:04.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:04.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:25:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:25:04.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.status 2024-09-14T07:25:04.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:25:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2024-09-14T07:25:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_32 2024-09-14T07:25:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_32 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_32.status 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_32.status 2024-09-14T07:25:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:04.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.status 2024-09-14T07:25:04.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2024-09-14T07:25:04.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.status 2024-09-14T07:25:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:25:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=2, entry_tid=3], [object_number=6, tag_tid=2, entry_tid=2]]], state=connected] 2024-09-14T07:25:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.status 2024-09-14T07:25:04.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:25:04.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:25:04.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:04.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2024-09-14T07:25:04.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:05.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2024-09-14T07:25:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_32': 2024-09-14T07:25:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:05.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0b13250c4 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0b13250c4 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:14:55 2024 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:14:55 2024 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:55 2024 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0b13250c4 2024-09-14T07:25:05.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:05.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:05.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: c43c6381-09d6-4313-bbfb-78a5290d6481 2024-09-14T07:25:05.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_32 snap 2024-09-14T07:25:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:25:05.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.export 2024-09-14T07:25:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_32.export 2024-09-14T07:25:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_32.export 2024-09-14T07:25:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.export 2024-09-14T07:25:05.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:05.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_32.export 2024-09-14T07:25:06.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:06.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_32.export 2024-09-14T07:25:06.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_32.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_32.export 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_33 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_33 snap 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:06.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2024-09-14T07:25:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:09.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2024-09-14T07:25:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:09.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:09.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:09.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:09.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-14T07:25:09.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T07:25:09.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:09.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:09.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:25:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:25:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:10.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:10.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:10.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:10.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-14T07:25:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T07:25:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:10.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:25:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:10.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:25:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:10.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:10.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:10.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-14T07:25:10.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T07:25:10.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:10.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:25:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:25:10.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:25:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:25:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:25:10.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:10.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:10.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:10.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2024-09-14T07:25:10.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:25:10.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:25:10.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:25:10.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:25:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:25:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_33 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_33 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:10.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:25:10.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:25:10.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:10.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:10.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_33 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_33 ']' 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_33' 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2024-09-14T07:25:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:10.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2024-09-14T07:25:10.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T07:25:10.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:10.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:10.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:10.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:10.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:10.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:10.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:10.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:10.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2024-09-14T07:25:10.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T07:25:10.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:10.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:10.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2024-09-14T07:25:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-14T07:25:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_33 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:25:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:11.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:25:11.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:25:11.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:25:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_33 2024-09-14T07:25:11.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:11.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_33 2024-09-14T07:25:11.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:11.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_33 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_33 '' 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_33 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_33.status 2024-09-14T07:25:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_33.status 2024-09-14T07:25:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:11.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.status 2024-09-14T07:25:11.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2024-09-14T07:25:11.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.status 2024-09-14T07:25:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6255], [object_number=6, tag_tid=1, entry_tid=6254]]], state=connected] 2024-09-14T07:25:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6255], [object_number=6, tag_tid=1, entry_tid=6254]]], state=connected] 2024-09-14T07:25:11.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.status 2024-09-14T07:25:11.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:11.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2024-09-14T07:25:11.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:11.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:11.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_33 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_33 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_33.status 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_33.status 2024-09-14T07:25:11.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:11.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.status 2024-09-14T07:25:11.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2024-09-14T07:25:11.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.status 2024-09-14T07:25:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6255], [object_number=6, tag_tid=1, entry_tid=6254]]], state=connected] 2024-09-14T07:25:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6255], [object_number=6, tag_tid=1, entry_tid=6254]]], state=connected] 2024-09-14T07:25:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.status 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:12.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_33': 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 12542170b14f 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.12542170b14f 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:14:28 2024 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:14:28 2024 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:28 2024 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 12542170b14f 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: cba0fe55-b477-4207-a602-c29bf1f9f942 2024-09-14T07:25:12.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:12.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_33 snap 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.export 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_33.export 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_33.export 2024-09-14T07:25:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.export 2024-09-14T07:25:12.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_33.export 2024-09-14T07:25:12.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:12.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_33.export 2024-09-14T07:25:12.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_33.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_33.export 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_34 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_34 snap 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2024-09-14T07:25:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:16.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:16.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:16.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:16.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:16.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:16.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:16.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2024-09-14T07:25:17.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:17.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:17.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:25:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:25:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:17.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:17.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:17.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:17.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2024-09-14T07:25:17.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:25:17.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:25:17.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:25:17.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:25:17.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:25:17.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:25:17.176 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_34 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_34 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:25:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:25:17.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:17.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:17.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_34 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_34 ']' 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_34' 2024-09-14T07:25:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:17.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:17.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:17.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:17.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:17.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:17.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:17.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_34 2024-09-14T07:25:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '' 2024-09-14T07:25:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:17.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:17.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:17.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:17.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:17.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:17.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:17.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:17.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:17.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:17.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:17.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:17.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:17.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:17.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:17.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:17.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:17.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9450' '!=' '' 2024-09-14T07:25:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:25:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9450' = '' 2024-09-14T07:25:17.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:17.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:17.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:17.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:17.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:25:17.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:17.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:17.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:17.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:25:17.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:17.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:17.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:17.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:25:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:17.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:17.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9450 2024-09-14T07:25:17.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9450 2024-09-14T07:25:17.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:17.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:17.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:17.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_34 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_34 ']' 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_34' 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:17.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:17.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:17.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:17.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:17.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:17.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:18.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:18.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:18.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:18.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:18.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2024-09-14T07:25:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:18.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:18.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:18.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:18.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:18.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:18.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:18.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_34 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '' 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:18.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:18.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:18.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:18.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:18.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:18.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:18.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:18.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:18.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:18.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:18.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:18.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:18.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:18.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:18.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:18.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:18.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:18.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:18.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:18.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9450' '!=' 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:18.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:18.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:25:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_34 2024-09-14T07:25:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_34 ']' 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_34' 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:18.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:18.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:18.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:18.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:18.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:18.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:18.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:18.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_34 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '' 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:19.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:19.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:19.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:19.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2024-09-14T07:25:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2024-09-14T07:25:19.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:19.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:19.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:19.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:19.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:19.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:19.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:19.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:19.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:19.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:19.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:19.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:19.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:19.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:19.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:19.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9450' '!=' 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:19.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:19.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:25:20.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4598765958590071) > weight (0.25) 2024-09-14T07:25:20.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7154469356116958) > weight (0.25) 2024-09-14T07:25:20.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3025267505787954) > weight (0.25) 2024-09-14T07:25:20.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7733337409047992) > weight (0.25) 2024-09-14T07:25:20.006 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-09-14T07:25:20.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_34 2024-09-14T07:25:20.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:20.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_34 ']' 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_34' 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:20.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:20.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:20.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:20.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:20.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:20.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:20.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:20.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_34 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '' 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:20.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:20.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:20.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:20.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:20.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:20.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:20.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:20.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:20.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:20.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:20.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:20.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:20.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2024-09-14T07:25:20.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:20.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:20.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:20.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:20.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:20.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:20.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:20.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:20.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:20.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:20.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9450' '!=' 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:20.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:20.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_34 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_34 ']' 2024-09-14T07:25:22.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_34' 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:22.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:22.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:22.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:22.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:22.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:22.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:22.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:22.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:22.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:22.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:22.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2024-09-14T07:25:22.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:22.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:22.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:22.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:22.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:22.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:22.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:22.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:22.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:22.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:22.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_34 2024-09-14T07:25:22.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:22.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:22.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:22.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '' 2024-09-14T07:25:22.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:22.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:22.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:22.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:22.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:22.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:22.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:22.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:22.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:22.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:22.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:22.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:22.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:22.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:22.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:22.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:22.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:22.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:22.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:22.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:22.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:22.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:22.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:22.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9450], [object_number=9, tag_tid=1, entry_tid=9449], [object_number=8, tag_tid=1, entry_tid=9448], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:22.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9450' '!=' 'object_number=10, tag_tid=1, entry_tid=9450' 2024-09-14T07:25:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:22.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:25:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_34 2024-09-14T07:25:24.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_34 ']' 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_34' 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:24.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:24.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2024-09-14T07:25:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:24.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:24.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:24.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:24.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:24.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:24.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-14T07:25:24.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:24.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_34 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:24.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:24.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:24.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:24.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_34 2024-09-14T07:25:24.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_34 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '' 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:24.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:24.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:24.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:24.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:24.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:24.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:24.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:24.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:24.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:25:24.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:24.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_34.status 2024-09-14T07:25:24.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_34.status 2024-09-14T07:25:24.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:24.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:24.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2024-09-14T07:25:24.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:25:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:25:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9450], [object_number=11, tag_tid=1, entry_tid=9447]]], state=connected] 2024-09-14T07:25:24.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.status 2024-09-14T07:25:24.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:24.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:25.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2024-09-14T07:25:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_34': 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 12548098679c 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.12548098679c 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:14:37 2024 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:14:37 2024 2024-09-14T07:25:25.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:15:38 2024 2024-09-14T07:25:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 12548098679c 2024-09-14T07:25:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 94e1f5e8-e95f-41c6-a4ca-2dcea846b13d 2024-09-14T07:25:25.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_34 snap 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.export 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_34.export 2024-09-14T07:25:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_34.export 2024-09-14T07:25:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.export 2024-09-14T07:25:26.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:26.116 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_34.export 2024-09-14T07:25:26.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:26.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_34.export 2024-09-14T07:25:26.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_34.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_34.export 2024-09-14T07:25:26.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:26.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_35 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_35 snap 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2024-09-14T07:25:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:29.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2024-09-14T07:25:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:29.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:29.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:29.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:29.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:29.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:29.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:29.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:29.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2024-09-14T07:25:30.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:30.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:30.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2024-09-14T07:25:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:25:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:30.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:30.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:30.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:30.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:30.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2024-09-14T07:25:30.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:30.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:30.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:30.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_35 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:30.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:30.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:30.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:25:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:30.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:30.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:30.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:30.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_35 2024-09-14T07:25:30.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:25:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:25:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:25:30.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:25:30.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_35 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_35 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_35 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_35 ']' 2024-09-14T07:25:30.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_35' 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:30.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:30.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2024-09-14T07:25:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:30.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:30.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:30.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:30.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:30.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:30.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:30.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:30.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:30.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:30.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2024-09-14T07:25:30.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:30.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:30.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:30.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:30.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:30.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:30.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:30.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:30.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:30.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2024-09-14T07:25:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:30.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:30.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:30.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:30.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:30.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:30.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:30.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:30.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_35 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '' 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:31.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:31.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:31.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:31.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098], [object_number=5, tag_tid=1, entry_tid=7097], [object_number=4, tag_tid=1, entry_tid=7096]]], state=connected] 2024-09-14T07:25:31.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:31.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:31.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:31.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:31.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098], [object_number=5, tag_tid=1, entry_tid=7097], [object_number=4, tag_tid=1, entry_tid=7096]]], state=connected] 2024-09-14T07:25:31.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7099' '!=' '' 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7099' = '' 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:25:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:31.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:25:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:25:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:31.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7099 2024-09-14T07:25:31.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7099 2024-09-14T07:25:31.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:31.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:31.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_35 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_35 ']' 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_35' 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:31.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:31.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:31.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:31.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:31.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:31.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:31.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:31.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:31.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:31.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:31.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:31.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:31.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:31.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:31.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:31.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:31.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:31.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:31.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:31.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:31.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:31.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:31.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:31.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_35 2024-09-14T07:25:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '' 2024-09-14T07:25:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:31.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:31.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:31.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098], [object_number=5, tag_tid=1, entry_tid=7097], [object_number=4, tag_tid=1, entry_tid=7096]]], state=connected] 2024-09-14T07:25:31.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:31.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:31.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:31.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:31.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:31.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098], [object_number=5, tag_tid=1, entry_tid=7097], [object_number=4, tag_tid=1, entry_tid=7096]]], state=connected] 2024-09-14T07:25:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7099' '!=' 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_35 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_35 ']' 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_35' 2024-09-14T07:25:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:32.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:32.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:32.273 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:32.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:32.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:32.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:32.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:32.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:32.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:32.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:32.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:32.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:32.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:32.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:32.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:32.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:32.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:32.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:32.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:32.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:32.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_35 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '' 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:32.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:32.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:32.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:32.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098], [object_number=5, tag_tid=1, entry_tid=7097], [object_number=4, tag_tid=1, entry_tid=7096]]], state=connected] 2024-09-14T07:25:32.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2024-09-14T07:25:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:32.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:32.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:32.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2024-09-14T07:25:32.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:32.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:32.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:32.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098], [object_number=5, tag_tid=1, entry_tid=7097], [object_number=4, tag_tid=1, entry_tid=7096]]], state=connected] 2024-09-14T07:25:32.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:32.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:32.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:32.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=7099' '!=' 'object_number=7, tag_tid=1, entry_tid=7099' 2024-09-14T07:25:32.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:32.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:25:33.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_35 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_35 ']' 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_35' 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:33.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:33.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:33.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:33.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:33.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:33.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:33.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:33.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:33.640 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:33.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:33.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:33.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:33.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:33.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:33.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_35 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:33.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:33.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:33.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:33.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:33.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-14T07:25:33.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:33.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:33.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:33.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_35 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:33.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:33.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:33.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:33.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:33.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_35 2024-09-14T07:25:34.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_35 2024-09-14T07:25:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '' 2024-09-14T07:25:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:34.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:34.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:34.079 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:34.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:34.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:34.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:34.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:34.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:34.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:34.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:34.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_35.status 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_35.status 2024-09-14T07:25:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2024-09-14T07:25:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:34.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:34.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:34.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:34.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:34.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7099], [object_number=6, tag_tid=1, entry_tid=7098]]], state=connected] 2024-09-14T07:25:34.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.status 2024-09-14T07:25:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:34.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2024-09-14T07:25:35.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_35': 2024-09-14T07:25:35.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:35.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:35.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:35.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11edbeed1ad1 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11edbeed1ad1 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:14:49 2024 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:14:49 2024 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:14:49 2024 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11edbeed1ad1 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:35.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 62aab55f-6928-44db-9f9e-e9bef3df1dd9 2024-09-14T07:25:35.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:35.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:35.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_35 snap 2024-09-14T07:25:35.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:35.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:25:35.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:35.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:35.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.export 2024-09-14T07:25:35.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_35.export 2024-09-14T07:25:35.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_35.export 2024-09-14T07:25:35.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.export 2024-09-14T07:25:35.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:35.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_35.export 2024-09-14T07:25:35.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:35.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_35.export 2024-09-14T07:25:35.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_35.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_35.export 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_36 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_36 snap 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:35.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:35.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:38.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2024-09-14T07:25:38.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:38.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:38.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2024-09-14T07:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:39.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2024-09-14T07:25:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:39.931 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:39.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2024-09-14T07:25:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:40.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:40.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2024-09-14T07:25:40.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:25:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:40.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:40.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:25:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:40.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2024-09-14T07:25:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:40.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2024-09-14T07:25:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:25:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:40.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:40.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:40.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:40.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:40.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:40.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:40.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:40.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:40.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:40.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:40.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:25:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:25:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:40.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:40.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:25:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:25:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:40.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:40.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:40.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:40.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2024-09-14T07:25:40.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:40.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:25:40.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:40.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:25:40.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:25:40.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:40.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:40.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:25:40.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:25:40.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:25:40.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:40.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:40.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:40.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2024-09-14T07:25:40.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:25:40.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:25:40.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:25:40.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:25:40.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:25:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:25:40.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_36 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_36 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:40.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:25:40.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:25:40.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:40.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:40.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_36 2024-09-14T07:25:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_36 ']' 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_36' 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:40.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:40.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:40.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:40.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:40.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:40.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:40.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_36 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:40.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:40.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_36 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:41.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:41.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_36 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:25:41.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:41.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:41.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:41.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:41.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:41.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_36 2024-09-14T07:25:41.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_36 '' 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_36.status 2024-09-14T07:25:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:41.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_36.status 2024-09-14T07:25:41.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2024-09-14T07:25:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:41.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542]]], state=connected] 2024-09-14T07:25:41.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542], [object_number=5, tag_tid=1, entry_tid=5541], [object_number=4, tag_tid=1, entry_tid=5540]]], state=connected] 2024-09-14T07:25:41.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:41.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2024-09-14T07:25:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_36.status 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_36.status 2024-09-14T07:25:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2024-09-14T07:25:41.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:41.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:41.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:41.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:41.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542]]], state=connected] 2024-09-14T07:25:41.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542], [object_number=5, tag_tid=1, entry_tid=5541], [object_number=4, tag_tid=1, entry_tid=5540]]], state=connected] 2024-09-14T07:25:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=7, tag_tid=1, entry_tid=5543' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=7, tag_tid=1, entry_tid=5543' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=5543' '!=' '' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=7, tag_tid=1, entry_tid=5543' = '' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=7, tag_tid=1, entry_tid=5543' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:41.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:41.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:25:41.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=5543' 2024-09-14T07:25:41.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:41.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:41.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:25:41.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:41.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:41.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:41.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:25:41.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=7, tag_tid=1, entry_tid=5543' 2024-09-14T07:25:41.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:41.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:41.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=5543 2024-09-14T07:25:41.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 5543 2024-09-14T07:25:41.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:41.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:41.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_36 2024-09-14T07:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_36 ']' 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_36' 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2024-09-14T07:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:41.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:41.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:41.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:41.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:41.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_36 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:25:41.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:25:41.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:41.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:41.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:25:41.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:25:41.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:25:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:25:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:41.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:41.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:41.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_36 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:41.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:25:41.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:41.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:41.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:25:41.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:41.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:25:41.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:25:41.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:41.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:41.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:25:41.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:25:41.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:25:41.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:25:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:42.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-14T07:25:42.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:25:42.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:25:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_36 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:25:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:25:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:25:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:25:42.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:25:42.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:42.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:42.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:25:42.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_36 2024-09-14T07:25:42.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_36 2024-09-14T07:25:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:42.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_36 '' 2024-09-14T07:25:42.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:42.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:42.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:42.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:42.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_36.status 2024-09-14T07:25:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_36.status 2024-09-14T07:25:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:42.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:42.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2024-09-14T07:25:42.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:42.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:42.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:42.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:42.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542]]], state=connected] 2024-09-14T07:25:42.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542]]], state=connected] 2024-09-14T07:25:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:42.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:25:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_36.status 2024-09-14T07:25:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_36.status 2024-09-14T07:25:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2024-09-14T07:25:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:42.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:42.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:42.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:42.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542]]], state=connected] 2024-09-14T07:25:42.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=5543], [object_number=6, tag_tid=1, entry_tid=5542]]], state=connected] 2024-09-14T07:25:42.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.status 2024-09-14T07:25:42.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:42.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:43.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2024-09-14T07:25:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_36': 2024-09-14T07:25:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1254983581c 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1254983581c 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:14:58 2024 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:14:58 2024 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:16:17 2024 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1254983581c 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:43.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: e56a071f-743d-4142-aa9d-144fee37ac0a 2024-09-14T07:25:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_36 snap 2024-09-14T07:25:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:25:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.export 2024-09-14T07:25:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_36.export 2024-09-14T07:25:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_36.export 2024-09-14T07:25:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.export 2024-09-14T07:25:43.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_36.export 2024-09-14T07:25:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_36.export 2024-09-14T07:25:43.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_36.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_36.export 2024-09-14T07:25:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_37 2024-09-14T07:25:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_37 snap 2024-09-14T07:25:43.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:46.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:46.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:47.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2024-09-14T07:25:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2024-09-14T07:25:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:47.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:47.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:47.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:47.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:47.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:47.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:47.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2024-09-14T07:25:47.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:25:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:25:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:25:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:25:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_37 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:47.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_37 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:48.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_37 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_37 ']' 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_37' 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2024-09-14T07:25:48.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:48.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:48.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:48.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:48.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2024-09-14T07:25:48.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_37 2024-09-14T07:25:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '' 2024-09-14T07:25:48.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:48.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:48.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:48.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:48.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:48.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:48.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:48.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:48.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:48.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:48.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:48.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7034' '!=' '' 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7034' = '' 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:48.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:25:48.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:48.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:25:48.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:48.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:25:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:48.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:48.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:25:48.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:48.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:25:48.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:25:48.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7034 2024-09-14T07:25:48.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7034 2024-09-14T07:25:48.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:25:48.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:48.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:25:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_37 2024-09-14T07:25:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_37 ']' 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_37' 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2024-09-14T07:25:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:48.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:48.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:48.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:48.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2024-09-14T07:25:48.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:48.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_37 2024-09-14T07:25:48.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '' 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:48.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:48.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:48.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:48.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:48.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:48.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:48.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:48.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:48.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2024-09-14T07:25:48.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:48.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:48.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:48.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7034' '!=' 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:25:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_37 2024-09-14T07:25:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_37 ']' 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_37' 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:49.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:49.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:49.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:49.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:49.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:49.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:49.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:49.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:49.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:49.443 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2024-09-14T07:25:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_37 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '' 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:49.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:49.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:49.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:49.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:49.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:49.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:49.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:49.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2024-09-14T07:25:49.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:49.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:49.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:49.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:49.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:49.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:49.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:49.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7034' '!=' 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:49.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:49.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:25:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_37 2024-09-14T07:25:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_37 ']' 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_37' 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:50.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:50.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:50.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:50.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:50.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:50.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:50.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:50.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:50.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:50.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:50.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2024-09-14T07:25:50.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_37 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '' 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:50.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:50.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:50.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:50.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2024-09-14T07:25:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:50.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7034' '!=' 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:50.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_37 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_37 ']' 2024-09-14T07:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_37' 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:52.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:52.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:52.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:52.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2024-09-14T07:25:52.517 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:52.518 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_37 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '' 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:52.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:52.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:52.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:52.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2024-09-14T07:25:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:52.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2024-09-14T07:25:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:52.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:52.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:52.579 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:52.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7034], [object_number=5, tag_tid=1, entry_tid=7033], [object_number=4, tag_tid=1, entry_tid=7032], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:52.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:52.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:52.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:52.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7034' '!=' 'object_number=6, tag_tid=1, entry_tid=7034' 2024-09-14T07:25:52.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:25:52.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_37 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_37 ']' 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_37' 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2024-09-14T07:25:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:54.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:25:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:25:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:25:54.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:25:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:54.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2024-09-14T07:25:54.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_37 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '' 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:54.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:25:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:54.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:54.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:25:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_37.status 2024-09-14T07:25:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_37.status 2024-09-14T07:25:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:25:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2024-09-14T07:25:54.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:54.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:25:54.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:25:54.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:25:54.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:54.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7034], [object_number=7, tag_tid=1, entry_tid=7031]]], state=connected] 2024-09-14T07:25:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.status 2024-09-14T07:25:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:25:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2024-09-14T07:25:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:54.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:25:54.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:55.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2024-09-14T07:25:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_37': 2024-09-14T07:25:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1182f25a3eac 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1182f25a3eac 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:15:07 2024 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:15:07 2024 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:15:07 2024 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:25:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1182f25a3eac 2024-09-14T07:25:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:25:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:25:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: abd7c614-a54c-41af-b917-4a6ab37bd0bb 2024-09-14T07:25:55.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:25:55.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:55.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_37 snap 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.export 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_37.export 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_37.export 2024-09-14T07:25:55.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.export 2024-09-14T07:25:56.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:56.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_37.export 2024-09-14T07:25:56.319 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:25:56.315+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:25:56.327 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:25:56.315+0000 7fea4ab92640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:25:56.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:25:56.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_37.export 2024-09-14T07:25:56.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_37.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_37.export 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_38 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_38 snap 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:25:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:25:58.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:25:59.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2024-09-14T07:25:59.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:25:59.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:25:59.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:25:59.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:25:59.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:25:59.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:25:59.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:25:59.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:25:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:25:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:25:59.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:25:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2024-09-14T07:26:00.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:00.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:00.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2024-09-14T07:26:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2024-09-14T07:26:00.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:26:00.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:26:00.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:26:00.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:26:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:26:00.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:26:00.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_38 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_38 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:00.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:26:00.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:26:00.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:26:00.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:00.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:26:00.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_38 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_38 ']' 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_38' 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:00.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:00.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:00.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:00.364 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:00.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:00.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:00.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2024-09-14T07:26:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:00.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:00.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:00.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:00.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_38 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '' 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:00.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:00.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:00.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:00.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:00.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:00.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:00.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:00.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:00.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:00.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:00.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:00.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:00.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:00.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=1, entry_tid=7445' '!=' '' 2024-09-14T07:26:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:26:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=1, entry_tid=7445' = '' 2024-09-14T07:26:00.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:00.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:00.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:26:00.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:00.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:26:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:26:00.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:00.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:26:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:26:00.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:26:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:26:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:00.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7445 2024-09-14T07:26:00.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7445 2024-09-14T07:26:00.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:26:00.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:00.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:26:01.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_38 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_38 ']' 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_38' 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:01.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:01.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:01.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:01.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:01.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:01.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:01.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:01.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:01.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:01.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:01.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:01.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:01.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:01.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:01.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:01.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:01.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:01.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_38 2024-09-14T07:26:01.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '' 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:01.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:01.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:01.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:01.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:01.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:01.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:01.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2024-09-14T07:26:01.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:01.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=1, entry_tid=7445' '!=' 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:26:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_38 2024-09-14T07:26:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:01.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_38 ']' 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_38' 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:01.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:01.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:01.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2024-09-14T07:26:01.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:02.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:02.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:02.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:02.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:02.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:02.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:02.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:02.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:02.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:02.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:02.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:02.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:02.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:02.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:02.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_38 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '' 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:02.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:02.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:02.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:02.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:02.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:02.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:02.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:02.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:02.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:02.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:02.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:02.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:02.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:02.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:02.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:02.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:02.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:02.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:02.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:02.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:02.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:02.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:02.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=1, entry_tid=7445' '!=' 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:02.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:02.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_38 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_38 ']' 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_38' 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:03.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:03.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:03.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:03.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:03.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:03.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:03.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:03.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:03.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:03.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:03.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:03.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:03.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2024-09-14T07:26:03.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:03.362 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:03.490 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_38 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '' 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:03.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:03.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:03.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:03.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:03.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:03.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:03.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:03.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:03.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:03.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2024-09-14T07:26:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2024-09-14T07:26:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:03.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:03.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:03.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:03.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:03.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:03.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:03.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:03.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:03.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=1, entry_tid=7445' '!=' 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:26:05.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_38 2024-09-14T07:26:05.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:05.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:05.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_38 ']' 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_38' 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:05.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:05.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:05.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2024-09-14T07:26:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:05.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:05.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:05.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:05.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:05.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:05.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:05.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:05.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:05.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:05.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:05.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2024-09-14T07:26:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:05.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:05.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:05.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:05.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:05.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_38 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '' 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:05.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:05.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:05.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:05.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:05.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:05.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=1, entry_tid=7445], [object_number=4, tag_tid=1, entry_tid=7444], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:05.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:05.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:05.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=1, entry_tid=7445' '!=' 'object_number=5, tag_tid=1, entry_tid=7445' 2024-09-14T07:26:05.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:05.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_38 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_38 ']' 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_38' 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2024-09-14T07:26:07.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2024-09-14T07:26:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:07.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:07.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:07.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:07.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:07.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:07.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:07.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:07.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:07.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:07.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:07.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-14T07:26:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_38 2024-09-14T07:26:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:07.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:07.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:07.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:07.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:07.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:07.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_38 2024-09-14T07:26:07.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_38 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '' 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:07.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:07.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:07.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:07.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:07.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:07.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2024-09-14T07:26:07.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:07.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:07.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:07.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2024-09-14T07:26:07.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:26:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_38.status 2024-09-14T07:26:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_38.status 2024-09-14T07:26:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:07.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:07.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2024-09-14T07:26:07.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:26:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:26:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=7443], [object_number=6, tag_tid=1, entry_tid=7442]]], state=connected] 2024-09-14T07:26:08.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.status 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:26:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:26:08.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:26:09.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2024-09-14T07:26:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_38': 2024-09-14T07:26:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b01e453b89 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b01e453b89 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:15:17 2024 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:15:17 2024 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:15:17 2024 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:26:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b01e453b89 2024-09-14T07:26:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:26:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:26:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 1e79f402-d2cc-4ece-9106-2d39b3c988a4 2024-09-14T07:26:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:26:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_38 snap 2024-09-14T07:26:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:26:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:26:09.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:26:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.export 2024-09-14T07:26:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_38.export 2024-09-14T07:26:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_38.export 2024-09-14T07:26:09.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.export 2024-09-14T07:26:09.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:26:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_38.export 2024-09-14T07:26:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:26:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_38.export 2024-09-14T07:26:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_38.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_38.export 2024-09-14T07:26:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:26:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_39 2024-09-14T07:26:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_39 snap 2024-09-14T07:26:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:26:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:26:09.868 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:26:09.865+0000 7fb3e494b640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 3 2024-09-14T07:26:11.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:26:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:26:12.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:26:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:26:12.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:12.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:12.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:12.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:12.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:12.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:12.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:12.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:12.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-14T07:26:13.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:13.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:13.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:13.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:13.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:13.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:13.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:13.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:13.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-14T07:26:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:13.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:13.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:13.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:13.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2024-09-14T07:26:13.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:26:13.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:26:13.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:26:13.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:26:13.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:26:13.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:26:13.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:26:13.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:13.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_39 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_39 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:26:13.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:13.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_39 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_39 ']' 2024-09-14T07:26:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_39' 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:13.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2024-09-14T07:26:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:13.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:13.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:13.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:13.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:13.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:13.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:13.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:13.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:13.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:13.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:13.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:13.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:13.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_39 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '' 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:13.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:13.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:13.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:13.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:14.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:14.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:14.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9642' '!=' '' 2024-09-14T07:26:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:26:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9642' = '' 2024-09-14T07:26:14.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:26:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:26:14.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:26:14.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:14.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:26:14.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:14.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:26:14.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:14.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:26:14.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:26:14.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:26:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=9642 2024-09-14T07:26:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 9642 2024-09-14T07:26:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:26:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_39 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_39 ']' 2024-09-14T07:26:14.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_39' 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:14.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:14.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:14.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:14.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:14.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:14.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:14.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:14.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:14.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:14.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:14.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:14.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2024-09-14T07:26:14.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:14.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:14.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:14.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:14.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:14.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2024-09-14T07:26:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:26:14.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:14.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:14.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:14.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:26:14.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:26:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:26:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:26:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:14.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:14.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:14.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:14.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:14.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:14.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:14.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:14.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_39 2024-09-14T07:26:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '' 2024-09-14T07:26:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:14.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:14.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:14.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:14.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:14.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.818 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:14.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:14.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:14.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:14.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:14.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:14.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:14.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:14.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:14.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:14.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9642' '!=' 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_39 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_39 ']' 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_39' 2024-09-14T07:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:15.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:15.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:15.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:15.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:15.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:15.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:15.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:15.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:15.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:15.421 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2024-09-14T07:26:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:15.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:15.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:15.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:15.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:15.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:15.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:15.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:15.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:15.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2024-09-14T07:26:15.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:26:15.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:26:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:26:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:26:15.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:15.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:15.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:26:15.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:15.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:15.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:15.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:15.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:15.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:15.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:15.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:15.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:15.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:15.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:15.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:15.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:15.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:15.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:15.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_39 2024-09-14T07:26:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '' 2024-09-14T07:26:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:15.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:15.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:15.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:15.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:15.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:15.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:15.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:15.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:15.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:15.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:15.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:15.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:15.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:15.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:15.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:15.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9642' '!=' 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_39 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_39 ']' 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_39' 2024-09-14T07:26:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:16.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:16.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:16.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:16.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:16.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:16.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:16.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:16.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:16.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:16.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:16.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:16.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:16.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:16.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:16.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:16.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:16.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:16.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2024-09-14T07:26:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:26:16.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:16.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:16.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:16.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:16.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:17.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_39 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '' 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:17.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:17.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:17.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:17.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:17.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:17.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:17.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:17.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:17.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:17.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2024-09-14T07:26:17.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:17.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:17.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:17.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:17.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:17.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:17.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:17.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:17.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=9642], [object_number=9, tag_tid=1, entry_tid=9641], [object_number=8, tag_tid=1, entry_tid=9640], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:17.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=9642' '!=' 'object_number=10, tag_tid=1, entry_tid=9642' 2024-09-14T07:26:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_39 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_39 ']' 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_39' 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:18.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:18.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2024-09-14T07:26:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:18.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:18.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:18.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:18.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:18.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:18.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2024-09-14T07:26:18.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:18.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:18.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:18.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:18.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:18.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:18.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:19.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-14T07:26:19.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:19.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:19.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:19.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_39 2024-09-14T07:26:19.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:26:19.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:19.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_39 2024-09-14T07:26:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_39 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '' 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:26:19.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:19.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:19.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:19.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:19.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:19.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:19.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:19.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:19.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:19.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:26:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_39.status 2024-09-14T07:26:19.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_39.status 2024-09-14T07:26:19.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:26:19.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:19.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2024-09-14T07:26:19.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:19.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:26:19.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:26:19.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:26:19.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:19.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=9642], [object_number=11, tag_tid=1, entry_tid=9639]]], state=connected] 2024-09-14T07:26:19.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.status 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:19.323 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:26:19.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:26:19.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:26:19.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:26:20.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2024-09-14T07:26:20.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_39': 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11edcc03ee33 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11edcc03ee33 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:15:28 2024 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:15:28 2024 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:16:29 2024 2024-09-14T07:26:20.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:26:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11edcc03ee33 2024-09-14T07:26:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:26:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:26:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 657c079d-f4b1-464b-aee3-b3f45750f06d 2024-09-14T07:26:20.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:26:20.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:20.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_39 snap 2024-09-14T07:26:20.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.export 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_39.export 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_39.export 2024-09-14T07:26:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.export 2024-09-14T07:26:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_39.export 2024-09-14T07:26:20.836 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:20.831+0000 7fea50b9e640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:26:20.836 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:20.831+0000 7fea4f39b640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:26:20.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:26:20.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_39.export 2024-09-14T07:26:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_39.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_39.export 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_40 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_40 snap 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:26:21.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:26:22.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6050863663828218) > weight (0.25) 2024-09-14T07:26:22.007 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T07:26:22.008 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T07:26:22.008 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:26:22.010 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:26:22.005+0000 7fc0883ce640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 102283) UID: 1000 2024-09-14T07:26:22.068 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:26:22.065+0000 7fc0813c0640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55755843a380 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2024-09-14T07:26:22.068 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:26:22.065+0000 7fc0813c0640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55755843a380 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2024-09-14T07:26:23.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:23.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:26:24.012 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T07:26:24.012 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T07:26:24.012 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-09-14T07:26:24.014 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:26:24.013+0000 7f77f27fc640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 99331) UID: 1000 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:26:24.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2024-09-14T07:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:24.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:24.927 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:24.928 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:25.013 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T07:26:25.014 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:26:25.014 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T07:26:25.014 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T07:26:25.014 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T07:26:25.017 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T07:26:25.017 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-09-14T07:26:25.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:25.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:25.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:25.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:25.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:25.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:25.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:25.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:25.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:25.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:25.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:25.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:25.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-14T07:26:25.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:25.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:25.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:26:25.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:25.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:26:25.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:26:25.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:25.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:26:25.334 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:26:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-14T07:26:25.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:25.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:25.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 1 2024-09-14T07:26:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:26.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:26.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:26.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:26.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:26.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:26.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:26.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:26.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:26.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:26:26.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:26.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-14T07:26:26.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:26.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:26.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:26.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:26.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:26.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:26.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:26.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:26.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:26.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-14T07:26:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-14T07:26:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:26:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:26.887 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:26:26.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:26:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:26:26.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:26:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-14T07:26:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:27.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:27.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 2 2024-09-14T07:26:29.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:29.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:29.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:29.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:29.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:29.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:29.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:29.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:29.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:29.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:29.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:29.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:29.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:29.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:29.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:29.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:29.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:29.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:29.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-14T07:26:29.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:29.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:29.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:29.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:29.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:29.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:29.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-14T07:26:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:29.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:26:29.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:26:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:26:29.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:26:29.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:29.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:29.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:29.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-14T07:26:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 4 2024-09-14T07:26:31.009 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T07:26:31.009 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:26:31.009 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T07:26:31.009 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T07:26:31.009 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T07:26:31.012 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T07:26:31.012 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing 2024-09-14T07:26:33.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:33.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:33.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:33.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:33.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:33.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:33.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:33.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:33.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:33.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:33.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:33.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:33.766 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:33.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-14T07:26:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:33.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:33.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:33.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:33.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:33.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:33.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:33.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:33.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:33.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:33.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-14T07:26:34.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:34.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:34.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:34.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:26:34.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:34.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:34.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:34.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:26:34.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:26:34.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:26:34.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:34.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:34.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:34.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-14T07:26:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T07:26:42.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:42.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:42.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:42.154 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:42.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:42.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:42.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:42.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:42.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:42.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:42.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:42.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:42.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:42.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:42.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:42.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:42.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:42.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:42.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:42.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:42.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:42.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-14T07:26:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:42.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:42.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:42.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:42.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:42.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:42.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-14T07:26:42.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:42.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:42.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:42.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:42.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-14T07:26:42.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:26:42.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:26:42.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:42.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:26:42.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:26:42.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:26:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-14T07:26:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:42.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:42.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:42.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T07:26:50.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:50.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:50.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:50.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:50.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:50.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:50.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:50.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:50.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:50.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:50.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:50.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:50.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:50.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:50.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:50.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:50.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:50.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:50.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:50.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:50.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:50.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:51.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:26:51.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:26:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:26:51.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:26:51.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:51.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:51.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:26:51.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-14T07:26:51.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:51.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:51.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:26:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:26:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:26:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:26:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:26:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:26:51.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-14T07:26:51.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:51.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:51.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 8 2024-09-14T07:26:53.700 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:26:53.700+0000 7fb3e8953640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:26:54.647 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.646+0000 7fea5efdb640 -1 reset not still connected to 0x55844520aa90 2024-09-14T07:26:54.647 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.646+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:26:54.648 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.646+0000 7fea5efdb640 -1 reset not still connected to 0x558446184b60 2024-09-14T07:26:54.654 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.654+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7948f0 2024-09-14T07:26:54.654 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.654+0000 7fea5efdb640 -1 reset not still connected to 0x55846479d520 2024-09-14T07:26:54.654 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.654+0000 7fea5efdb640 -1 reset not still connected to 0x5584647a31e0 2024-09-14T07:26:54.654 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:26:54.654+0000 7fea5efdb640 -1 reset not still connected to 0x558471f1a000 2024-09-14T07:26:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:59.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:59.281 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:59.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:59.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:26:59.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:59.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:59.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:59.282 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:26:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:59.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:59.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:59.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:59.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-14T07:26:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:26:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-14T07:26:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:26:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:26:59.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:26:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:26:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:59.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:59.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:59.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:59.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-14T07:26:59.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:26:59.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:26:59.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:26:59.584 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:26:59.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:26:59.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:26:59.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:26:59.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:26:59.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_40 2024-09-14T07:26:59.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:26:59.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_40 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:26:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:26:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_40 2024-09-14T07:26:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_40 ']' 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_40' 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:26:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:26:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:26:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:26:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:26:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:59.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:26:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:26:59.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:59.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:59.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:26:59.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2024-09-14T07:26:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:26:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:26:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:26:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:26:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2024-09-14T07:26:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:26:59.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:26:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:26:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:26:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:26:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:26:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:26:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:26:59.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:26:59.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_40 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '' 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:00.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:00.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:00.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:00.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:00.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2024-09-14T07:27:00.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7834' '!=' '' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7834' = '' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:00.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:00.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:27:00.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:00.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:00.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:27:00.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:00.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:00.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:27:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:00.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=7834 2024-09-14T07:27:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 7834 2024-09-14T07:27:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_40 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_40 ']' 2024-09-14T07:27:00.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_40' 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:00.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:00.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:00.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:00.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:00.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:00.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:00.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:00.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:27:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2024-09-14T07:27:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:00.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:00.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:00.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_40 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '' 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:00.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:00.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:00.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:00.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:00.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:00.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:00.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:00.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2024-09-14T07:27:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2024-09-14T07:27:00.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:00.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:00.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:00.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:00.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:00.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7834' '!=' 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:27:01.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_40 2024-09-14T07:27:01.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_40 ']' 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_40' 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:01.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:01.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:01.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:01.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:01.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:01.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:01.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:01.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:01.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:01.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:01.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:01.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:01.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:01.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:01.249 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:27:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:01.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:01.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:01.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:01.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:01.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:01.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:01.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:01.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:01.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:01.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:01.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_40 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '' 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:01.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:01.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:01.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:01.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:01.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:01.515 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:01.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:01.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:01.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:01.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:01.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:01.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:01.571 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:01.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:01.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:01.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:01.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7834' '!=' 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:01.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:01.624 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:27:02.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_40 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_40 ']' 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_40' 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:02.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:02.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:02.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2024-09-14T07:27:02.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:02.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:02.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:02.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:02.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:02.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:02.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:02.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:02.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:02.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:02.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:02.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:02.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:27:02.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:02.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:02.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:02.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2024-09-14T07:27:02.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:02.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:02.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:02.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:02.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:02.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:02.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:02.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:02.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_40 2024-09-14T07:27:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '' 2024-09-14T07:27:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:02.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:02.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:02.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:02.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:02.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:02.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:02.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:02.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:02.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:02.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=6, tag_tid=1, entry_tid=7834], [object_number=5, tag_tid=1, entry_tid=7833], [object_number=4, tag_tid=1, entry_tid=7832], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=6, tag_tid=1, entry_tid=7834' '!=' 'object_number=6, tag_tid=1, entry_tid=7834' 2024-09-14T07:27:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_40 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_40 ']' 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_40' 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:04.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:04.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:04.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:04.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:04.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:04.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:04.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:04.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:04.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:04.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:04.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-14T07:27:04.572 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:04.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2024-09-14T07:27:04.748 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:04.749 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_40 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '' 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:04.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:04.751 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:04.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:04.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:04.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:04.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2024-09-14T07:27:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_40.status 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_40.status 2024-09-14T07:27:04.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:04.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:04.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2024-09-14T07:27:04.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=6, tag_tid=1, entry_tid=7834], [object_number=7, tag_tid=1, entry_tid=7831]]], state=connected] 2024-09-14T07:27:04.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.status 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:04.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:27:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:04.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:05.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2024-09-14T07:27:05.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_40': 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 12542de82c59 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.12542de82c59 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:05.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:15:40 2024 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:15:40 2024 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:16:41 2024 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 12542de82c59 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 6e4d0b7b-83a4-484c-af06-4104d087775c 2024-09-14T07:27:05.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_40 snap 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.export 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_40.export 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_40.export 2024-09-14T07:27:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.export 2024-09-14T07:27:06.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_40.export 2024-09-14T07:27:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:06.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_40.export 2024-09-14T07:27:06.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_40.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_40.export 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_41 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_41 snap 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:06.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2024-09-14T07:27:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2024-09-14T07:27:07.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:07.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:07.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:07.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:07.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:07.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:07.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:08.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2024-09-14T07:27:08.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:08.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:08.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:08.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:08.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2024-09-14T07:27:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:08.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:08.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:08.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:08.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2024-09-14T07:27:09.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T07:27:09.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:09.013 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:09.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:09.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2024-09-14T07:27:09.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:09.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:09.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:09.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:09.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:09.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:09.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:27:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:27:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:09.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:09.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:09.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:09.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2024-09-14T07:27:09.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:27:09.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:27:09.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:27:09.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:27:09.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:27:09.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:27:09.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_41 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_41 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:27:09.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:27:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:09.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:09.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_41 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_41 ']' 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_41' 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2024-09-14T07:27:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:09.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:09.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:09.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:09.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T07:27:09.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:09.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:09.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:09.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:09.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:09.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:09.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:09.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:09.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:09.675 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_41 2024-09-14T07:27:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '' 2024-09-14T07:27:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:09.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:09.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:09.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:09.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2024-09-14T07:27:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:09.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:09.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:09.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:09.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:09.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:09.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11610' '!=' '' 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11610' = '' 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:09.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:09.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:09.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:27:09.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:09.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:09.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:09.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:27:09.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:09.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:09.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:27:09.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:09.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:09.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:09.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11610 2024-09-14T07:27:09.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11610 2024-09-14T07:27:09.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:09.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:09.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:09.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_41 2024-09-14T07:27:09.995 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_41 ']' 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_41' 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:09.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2024-09-14T07:27:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:09.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:10.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:10.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:10.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:10.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:10.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:10.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T07:27:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2024-09-14T07:27:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:10.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:10.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:10.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:10.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:10.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_41 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '' 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:10.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:10.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:10.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:10.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:10.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:10.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:10.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2024-09-14T07:27:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:10.351 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:10.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:10.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:10.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:10.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:10.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:10.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:10.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:10.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11610' '!=' 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:10.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:10.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:27:10.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_41 2024-09-14T07:27:10.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:10.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:10.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_41 ']' 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_41' 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:10.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:10.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:10.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:10.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2024-09-14T07:27:10.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:10.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:10.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T07:27:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:10.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:10.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:10.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:10.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:11.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_41 2024-09-14T07:27:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '' 2024-09-14T07:27:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:11.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:11.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:11.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:11.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:11.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:11.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:11.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:11.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11610' '!=' 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:11.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:11.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:27:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_41 2024-09-14T07:27:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_41 ']' 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_41' 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:12.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:12.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:12.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:12.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:12.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:12.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:12.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:12.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T07:27:12.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:12.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:12.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:12.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:12.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:12.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:12.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:12.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:12.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:12.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_41 2024-09-14T07:27:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '' 2024-09-14T07:27:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:12.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:12.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:12.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:12.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:12.374 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:12.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:12.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:12.380 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:12.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:12.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=10, tag_tid=1, entry_tid=11610], [object_number=9, tag_tid=1, entry_tid=11609], [object_number=8, tag_tid=1, entry_tid=11608], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:12.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=10, tag_tid=1, entry_tid=11610' '!=' 'object_number=10, tag_tid=1, entry_tid=11610' 2024-09-14T07:27:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:27:14.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_41 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_41 ']' 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_41' 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:14.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:14.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:14.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:14.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:14.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:14.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:14.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:14.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:14.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-14T07:27:14.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:14.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2024-09-14T07:27:14.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:14.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:14.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2024-09-14T07:27:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_41 2024-09-14T07:27:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '' 2024-09-14T07:27:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:14.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:14.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:14.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:14.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:14.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:14.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:14.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:14.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:14.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2024-09-14T07:27:14.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:14.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:14.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:14.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2024-09-14T07:27:14.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:14.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:14.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:27:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_41.status 2024-09-14T07:27:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_41.status 2024-09-14T07:27:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2024-09-14T07:27:14.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=10, tag_tid=1, entry_tid=11610], [object_number=11, tag_tid=1, entry_tid=11607]]], state=connected] 2024-09-14T07:27:14.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.status 2024-09-14T07:27:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2024-09-14T07:27:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2024-09-14T07:27:15.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_41': 2024-09-14T07:27:15.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:15.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:15.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:15.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1182303c393 2024-09-14T07:27:15.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1182303c393 2024-09-14T07:27:15.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:15.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:15.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:15:50 2024 2024-09-14T07:27:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:15:50 2024 2024-09-14T07:27:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:16:51 2024 2024-09-14T07:27:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1182303c393 2024-09-14T07:27:15.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:15.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:15.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 055eb9c2-662d-454b-b3b2-f7c8dfb5d945 2024-09-14T07:27:15.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:15.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:15.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_41 snap 2024-09-14T07:27:15.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:15.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:27:15.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:15.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:15.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.export 2024-09-14T07:27:15.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_41.export 2024-09-14T07:27:15.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_41.export 2024-09-14T07:27:15.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.export 2024-09-14T07:27:15.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:15.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_41.export 2024-09-14T07:27:15.928 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:27:15.925+0000 7fea4fb9c640 -1 bdev(0x55843bdacc00 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2024-09-14T07:27:16.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:16.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_41.export 2024-09-14T07:27:16.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_41.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_41.export 2024-09-14T07:27:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_42 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_42 snap 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2024-09-14T07:27:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2024-09-14T07:27:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:17.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:17.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:17.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:17.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:17.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:17.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:17.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:17.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:17.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:17.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2024-09-14T07:27:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:18.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:18.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:18.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:27:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:18.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:18.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:18.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:18.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2024-09-14T07:27:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:27:18.192 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:27:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:27:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:27:18.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:27:18.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_42 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:18.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_42 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:18.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_42 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_42 ']' 2024-09-14T07:27:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_42' 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:18.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:18.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:18.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:18.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:18.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:18.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:18.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:18.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:18.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:18.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:18.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:18.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:18.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:18.565 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:18.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_42 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '' 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:18.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:18.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:18.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:18.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:18.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:18.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:18.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:18.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:18.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:18.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:18.750 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:18.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:18.755 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:18.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:18.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:18.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:18.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' '' 2024-09-14T07:27:18.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:27:18.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' = '' 2024-09-14T07:27:18.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:18.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:18.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:18.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:27:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:18.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T07:27:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:18.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:27:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=1 2024-09-14T07:27:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 1 2024-09-14T07:27:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:18.814 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_42 2024-09-14T07:27:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_42 ']' 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_42' 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:19.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:19.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:19.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:19.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:19.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:19.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:19.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:19.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:19.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:19.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:19.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_42 2024-09-14T07:27:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '' 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:19.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:19.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:19.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:19.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:19.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:19.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:19.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:19.372 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:19.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:19.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:19.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:19.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:19.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:19.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:19.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:19.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:19.422 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:19.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:19.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:19.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:19.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:19.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:27:19.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_42 2024-09-14T07:27:19.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_42 ']' 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_42' 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:19.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:19.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:19.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:19.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:19.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:19.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:19.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:19.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:19.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:19.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:19.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_42 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '' 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:20.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:20.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:20.126 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:20.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:20.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:20.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:20.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2024-09-14T07:27:20.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:20.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:20.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:20.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:27:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_42 2024-09-14T07:27:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_42 ']' 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_42' 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:21.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:21.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:21.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:21.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:21.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:21.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:21.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:21.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:21.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_42 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '' 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:21.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:21.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:21.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:21.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:21.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:21.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:21.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:21.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:21.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:21.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:21.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:21.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:21.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:21.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:21.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:21.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:21.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:27:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_42 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_42 ']' 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_42' 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:23.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:23.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2024-09-14T07:27:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:23.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:23.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:23.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:23.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_42 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '' 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:23.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:23.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:23.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:23.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:23.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:23.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:23.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:23.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:23.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:23.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:23.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:23.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:23.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:27:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:27:25.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_42 2024-09-14T07:27:25.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_42 ']' 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_42' 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:25.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:25.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:25.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:25.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:25.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:25.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:25.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:25.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:25.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:25.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:25.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:25.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:25.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:25.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:25.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-14T07:27:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:25.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_42 2024-09-14T07:27:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:25.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:25.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:25.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:25.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:25.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:25.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:25.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:25.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_42 2024-09-14T07:27:25.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:25.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_42 2024-09-14T07:27:25.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:25.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:25.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:25.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '' 2024-09-14T07:27:25.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:25.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:25.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:25.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:25.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:25.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:25.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:25.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:25.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:25.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:25.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:25.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:25.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:25.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:25.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2024-09-14T07:27:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:25.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_42.status 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_42.status 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:25.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:25.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2024-09-14T07:27:25.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:25.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:25.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3595], [object_number=2, tag_tid=1, entry_tid=3594]]], state=connected] 2024-09-14T07:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.status 2024-09-14T07:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:26.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_42': 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 125e5bcec0e1 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.125e5bcec0e1 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:02 2024 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:02 2024 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:03 2024 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 125e5bcec0e1 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 6f0a591e-0388-4433-a339-b91507961c44 2024-09-14T07:27:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:26.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:26.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_42 snap 2024-09-14T07:27:26.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.export 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_42.export 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_42.export 2024-09-14T07:27:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.export 2024-09-14T07:27:27.017 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T07:27:27.018 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T07:27:27.019 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T07:27:27.019 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:27:27.019+0000 7f616b1b6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 116917) UID: 1000 2024-09-14T07:27:27.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:27.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_42.export 2024-09-14T07:27:27.417 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_42.export 2024-09-14T07:27:27.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_42.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_42.export 2024-09-14T07:27:27.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_43 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_43 snap 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:27.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:29.020 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T07:27:29.020 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:27:29.020 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T07:27:29.020 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T07:27:29.021 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T07:27:29.023 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T07:27:29.023 INFO:tasks.rbd_mirror_thrash:waiting for 63 secs before thrashing 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:30.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:30.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2024-09-14T07:27:30.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:30.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:30.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:30.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:30.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:30.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:30.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:30.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:30.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:30.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:30.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:30.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:30.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:30.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2024-09-14T07:27:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-14T07:27:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:31.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:31.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:31.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:31.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:31.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:27:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:27:31.065 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:31.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2024-09-14T07:27:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:27:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_43 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:27:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_43 2024-09-14T07:27:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:27:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:27:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_43 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_43 ']' 2024-09-14T07:27:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_43' 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:31.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:31.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2024-09-14T07:27:31.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:31.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:31.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-14T07:27:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:31.569 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:31.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:31.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2024-09-14T07:27:31.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:31.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:31.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:31.574 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:31.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:31.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:31.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:31.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:31.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:31.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_43 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '' 2024-09-14T07:27:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:31.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:31.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:31.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:31.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:31.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:31.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:31.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:31.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10504], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502], [object_number=9, tag_tid=1, entry_tid=10501]]], state=connected] 2024-09-14T07:27:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:31.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2024-09-14T07:27:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2024-09-14T07:27:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:31.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:31.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10504], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502], [object_number=9, tag_tid=1, entry_tid=10501]]], state=connected] 2024-09-14T07:27:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10504' '!=' '' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10504' = '' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:31.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:31.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:27:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:27:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:27:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:31.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:31.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=10504 2024-09-14T07:27:31.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 10504 2024-09-14T07:27:31.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:31.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:31.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_43 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_43 ']' 2024-09-14T07:27:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_43' 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:32.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:32.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:32.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-14T07:27:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:32.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2024-09-14T07:27:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:32.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:32.195 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:32.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:32.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:32.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:32.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:32.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:32.352 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:32.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_43 2024-09-14T07:27:32.353 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '' 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:32.354 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:32.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:32.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:32.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:32.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:32.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10504], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502], [object_number=9, tag_tid=1, entry_tid=10501]]], state=connected] 2024-09-14T07:27:32.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:32.625 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:32.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:32.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:32.628 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:32.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:32.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:32.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:32.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:32.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10504], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502], [object_number=9, tag_tid=1, entry_tid=10501]]], state=connected] 2024-09-14T07:27:32.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:32.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:32.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:32.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10504' '!=' 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:32.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:32.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:27:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_43 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_43 ']' 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_43' 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:33.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:33.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:33.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:33.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:33.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:33.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:33.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-14T07:27:33.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:33.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:33.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:33.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:33.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:33.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_43 2024-09-14T07:27:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:33.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '' 2024-09-14T07:27:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:33.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:33.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:33.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:33.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:33.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:33.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:33.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:33.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:33.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10504], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502], [object_number=9, tag_tid=1, entry_tid=10501]]], state=connected] 2024-09-14T07:27:33.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:33.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:33.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:33.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:33.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:33.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=8, tag_tid=1, entry_tid=10504], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502], [object_number=9, tag_tid=1, entry_tid=10501]]], state=connected] 2024-09-14T07:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:33.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:33.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:33.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=8, tag_tid=1, entry_tid=10504' '!=' 'object_number=8, tag_tid=1, entry_tid=10504' 2024-09-14T07:27:33.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:33.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:27:34.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_43 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_43 ']' 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_43' 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:34.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2024-09-14T07:27:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:34.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:34.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:34.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:34.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:34.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-14T07:27:34.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:34.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:34.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:34.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:34.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:34.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:34.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:34.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:34.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2024-09-14T07:27:34.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:34.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_43 2024-09-14T07:27:34.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:34.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:34.611 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:34.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '' 2024-09-14T07:27:34.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:34.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:34.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:34.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:34.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:34.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:34.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:34.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:34.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:34.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:34.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:34.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:34.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:34.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:34.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2024-09-14T07:27:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2024-09-14T07:27:34.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:27:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_43.status 2024-09-14T07:27:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_43.status 2024-09-14T07:27:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2024-09-14T07:27:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:27:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:27:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:34.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:34.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=10503], [object_number=10, tag_tid=1, entry_tid=10502]]], state=connected] 2024-09-14T07:27:34.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.status 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:34.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:35.726 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_43': 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 118277446072 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.118277446072 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:35.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:08 2024 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:08 2024 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:09 2024 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 118277446072 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: b0c2107d-c75c-43b8-94a7-cdbde4f99e37 2024-09-14T07:27:35.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:35.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:35.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_43 snap 2024-09-14T07:27:35.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:35.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:27:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.export 2024-09-14T07:27:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_43.export 2024-09-14T07:27:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_43.export 2024-09-14T07:27:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.export 2024-09-14T07:27:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:35.989 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_43.export 2024-09-14T07:27:36.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:36.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_43.export 2024-09-14T07:27:36.382 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_43.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_43.export 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_44 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_44 snap 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:36.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:37.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:37.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:37.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2024-09-14T07:27:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:38.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:38.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2024-09-14T07:27:38.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:38.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:38.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:38.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2024-09-14T07:27:39.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:27:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:27:39.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:27:39.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:27:39.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:27:39.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:27:39.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:27:39.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:39.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_44 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_44 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:27:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:27:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_44 2024-09-14T07:27:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_44 ']' 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_44' 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:39.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2024-09-14T07:27:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:39.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2024-09-14T07:27:39.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_44 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_44 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_44 '' 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_44 2024-09-14T07:27:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_44.status 2024-09-14T07:27:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_44.status 2024-09-14T07:27:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:39.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.status 2024-09-14T07:27:39.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2024-09-14T07:27:39.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.status 2024-09-14T07:27:39.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:39.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:39.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:39.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6607], [object_number=6, tag_tid=1, entry_tid=6606]]], state=connected] 2024-09-14T07:27:39.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6607], [object_number=6, tag_tid=1, entry_tid=6606]]], state=connected] 2024-09-14T07:27:39.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.status 2024-09-14T07:27:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:39.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2024-09-14T07:27:39.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:39.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_44 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_44 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_44.status 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_44.status 2024-09-14T07:27:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.status 2024-09-14T07:27:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2024-09-14T07:27:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.status 2024-09-14T07:27:39.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:39.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:39.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:39.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6607], [object_number=6, tag_tid=1, entry_tid=6606]]], state=connected] 2024-09-14T07:27:39.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6607], [object_number=6, tag_tid=1, entry_tid=6606]]], state=connected] 2024-09-14T07:27:39.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.status 2024-09-14T07:27:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=2, entry_tid=1' -a 'object_number=5, tag_tid=2, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:39.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2024-09-14T07:27:40.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_44': 2024-09-14T07:27:40.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:40.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 125e72a388b9 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.125e72a388b9 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:21 2024 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:21 2024 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:16:21 2024 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 125e72a388b9 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:40.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 2401fb36-6a0b-4f8a-9963-765460fe8bd2 2024-09-14T07:27:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_44 snap 2024-09-14T07:27:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:27:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:40.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:40.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.export 2024-09-14T07:27:40.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_44.export 2024-09-14T07:27:40.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_44.export 2024-09-14T07:27:40.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.export 2024-09-14T07:27:40.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_44.export 2024-09-14T07:27:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:41.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_44.export 2024-09-14T07:27:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_44.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_44.export 2024-09-14T07:27:41.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:41.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:41.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_45 2024-09-14T07:27:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_45 snap 2024-09-14T07:27:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:41.351 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:27:41.351+0000 7fb3ea156640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 8 2024-09-14T07:27:41.353 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:27:41.351+0000 7fb3e7150640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 6 2024-09-14T07:27:41.354 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:27:41.355+0000 7fb3e494b640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 7 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:41.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:41.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:41.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:41.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:42.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2024-09-14T07:27:42.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:42.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:42.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:42.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:42.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:42.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:42.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2024-09-14T07:27:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:42.599 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:42.600 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:42.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:42.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:42.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:42.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:42.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:42.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:42.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:42.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:42.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:42.623 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2024-09-14T07:27:42.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T07:27:42.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:42.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2024-09-14T07:27:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:42.743 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:27:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:27:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:42.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:42.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:42.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:42.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2024-09-14T07:27:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T07:27:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2024-09-14T07:27:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:27:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:27:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:27:42.883 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2024-09-14T07:27:43.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T07:27:43.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:43.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_45 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:27:43.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:27:43.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:27:43.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:27:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:27:43.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:27:43.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:27:43.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:27:43.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:27:43.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_45 2024-09-14T07:27:43.178 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:27:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:27:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:27:43.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:27:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_45 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_45 2024-09-14T07:27:43.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_45 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_45 ']' 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_45' 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:43.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2024-09-14T07:27:43.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T07:27:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2024-09-14T07:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:43.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:43.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:43.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:43.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2024-09-14T07:27:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T07:27:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:27:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:27:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:27:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:43.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2024-09-14T07:27:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-09-14T07:27:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:27:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:27:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:27:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:27:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:27:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:43.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:27:43.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:27:43.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:27:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:27:43.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:27:43.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:27:43.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2024-09-14T07:27:43.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_45 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_45 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_45 '' 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_45 2024-09-14T07:27:43.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:43.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_45.status 2024-09-14T07:27:43.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_45.status 2024-09-14T07:27:43.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:43.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.status 2024-09-14T07:27:43.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2024-09-14T07:27:43.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.status 2024-09-14T07:27:44.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:44.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:44.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:44.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3963], [object_number=2, tag_tid=1, entry_tid=3962]]], state=connected] 2024-09-14T07:27:44.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3963], [object_number=2, tag_tid=1, entry_tid=3962]]], state=connected] 2024-09-14T07:27:44.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.status 2024-09-14T07:27:44.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:44.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2024-09-14T07:27:44.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:44.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:44.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_45 2024-09-14T07:27:44.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_45 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_45.status 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_45.status 2024-09-14T07:27:44.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:44.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.status 2024-09-14T07:27:44.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2024-09-14T07:27:44.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.status 2024-09-14T07:27:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:27:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:27:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3963], [object_number=2, tag_tid=1, entry_tid=3962]]], state=connected] 2024-09-14T07:27:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3963], [object_number=2, tag_tid=1, entry_tid=3962]]], state=connected] 2024-09-14T07:27:44.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.status 2024-09-14T07:27:44.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:44.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=1' -a 'object_number=1, tag_tid=3, entry_tid=1' = 'object_number=1, tag_tid=3, entry_tid=1' 2024-09-14T07:27:44.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:44.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:45.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_45': 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1182e78e8882 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1182e78e8882 2024-09-14T07:27:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:30 2024 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:30 2024 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:31 2024 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1182e78e8882 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: f03cb828-fd1c-4b11-b9e4-529c50463d29 2024-09-14T07:27:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:45.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:45.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_45 snap 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.export 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_45.export 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_45.export 2024-09-14T07:27:45.152 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.export 2024-09-14T07:27:45.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:45.321 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_45.export 2024-09-14T07:27:45.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_45.export 2024-09-14T07:27:45.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_45.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_45.export 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_46 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_46 snap 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:45.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:45.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:46.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:46.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:46.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2024-09-14T07:27:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:47.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:47.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:47.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:47.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:47.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:47.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:47.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:47.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2024-09-14T07:27:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:47.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2024-09-14T07:27:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:27:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:27:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2024-09-14T07:27:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:27:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:48.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:48.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:48.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:27:48.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:27:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:48.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:48.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:48.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2024-09-14T07:27:48.274 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:27:48.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:27:48.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_46 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:48.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_46 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_46 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_46 ']' 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_46' 2024-09-14T07:27:48.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:48.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:48.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:48.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2024-09-14T07:27:48.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:48.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:48.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:48.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:48.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:48.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:48.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:48.629 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:48.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:48.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:48.769 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2024-09-14T07:27:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:27:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:27:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:48.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_46 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '' 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:48.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:48.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:48.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:48.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:48.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:48.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:48.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:48.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:48.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:48.984 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:48.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2024-09-14T07:27:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:49.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:49.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:49.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:49.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:49.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' '' 2024-09-14T07:27:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:27:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' = '' 2024-09-14T07:27:49.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:49.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:49.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:49.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=3 2024-09-14T07:27:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:27:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=2 2024-09-14T07:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:27:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:27:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:27:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=1 2024-09-14T07:27:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 3 = 2 -a 1 -le 1 2024-09-14T07:27:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:27:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:27:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_46 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_46 ']' 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_46' 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:49.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:49.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:49.279 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:49.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:49.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:49.280 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:49.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:49.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:49.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:49.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:49.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:49.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:27:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:49.528 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:49.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:49.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_46 2024-09-14T07:27:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '' 2024-09-14T07:27:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:49.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:49.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.738 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:49.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:49.740 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:49.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:49.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:49.788 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:49.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:27:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_46 2024-09-14T07:27:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_46 ']' 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_46' 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2024-09-14T07:27:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:50.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:50.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:50.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2024-09-14T07:27:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:50.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:50.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:50.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:50.360 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:50.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:50.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:50.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:50.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:50.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:27:50.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:27:50.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:50.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:50.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:50.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:50.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:50.630 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:50.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_46 2024-09-14T07:27:50.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:50.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:50.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:50.631 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '' 2024-09-14T07:27:50.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:50.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:50.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:50.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:50.632 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:50.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:50.633 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:50.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:50.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:50.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:50.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:50.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:50.682 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:50.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:50.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:50.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2024-09-14T07:27:50.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:50.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:50.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:50.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:50.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:50.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:50.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:50.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:50.736 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:50.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:50.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:50.737 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:50.739 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:50.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_46 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_46 ']' 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_46' 2024-09-14T07:27:51.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:51.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:51.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:51.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:51.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:51.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:51.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:51.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:51.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:51.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:51.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:51.575 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:51.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:51.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:51.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2024-09-14T07:27:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:51.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:51.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:51.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:51.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:51.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:51.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:51.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:51.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2024-09-14T07:27:51.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:27:51.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:51.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:51.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:51.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:51.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:51.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:51.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_46 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '' 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:52.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:52.031 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:52.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:52.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:52.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:52.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:52.084 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=2, entry_tid=1], [object_number=4, tag_tid=2, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:52.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:52.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:52.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:52.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=5, tag_tid=2, entry_tid=1' '!=' 'object_number=5, tag_tid=2, entry_tid=1' 2024-09-14T07:27:52.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:27:52.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:27:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_46 2024-09-14T07:27:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_46 ']' 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_46' 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:27:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:27:53.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:53.695 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:53.696 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:53.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:53.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:53.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:53.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:53.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:53.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:53.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:53.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:53.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:53.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:53.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:27:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:53.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:27:53.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:27:53.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:53.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:27:53.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:27:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:53.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:27:53.871 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:53.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-09-14T07:27:53.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:27:53.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:27:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:27:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:27:53.981 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:27:54.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:54.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:54.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:27:54.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2024-09-14T07:27:54.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_46 2024-09-14T07:27:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '' 2024-09-14T07:27:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:54.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:54.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:54.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:27:54.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:54.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:54.142 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:54.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:54.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:54.143 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:54.190 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:54.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:54.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:54.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:54.191 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:54.194 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:54.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:54.197 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2024-09-14T07:27:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2024-09-14T07:27:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:27:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:27:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:27:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:27:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_46.status 2024-09-14T07:27:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_46.status 2024-09-14T07:27:54.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:27:54.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:54.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2024-09-14T07:27:54.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:54.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:27:54.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:27:54.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:27:54.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:54.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6307], [object_number=6, tag_tid=1, entry_tid=6306]]], state=connected] 2024-09-14T07:27:54.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.status 2024-09-14T07:27:54.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:54.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:27:54.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:54.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2024-09-14T07:27:54.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:27:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:55.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_46': 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0ea751ec6 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0ea751ec6 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:27:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:35 2024 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:35 2024 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:36 2024 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0ea751ec6 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: fc6d034c-19f4-4f04-a5c8-414c007b202c 2024-09-14T07:27:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_46 snap 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.export 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_46.export 2024-09-14T07:27:55.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_46.export 2024-09-14T07:27:55.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.export 2024-09-14T07:27:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:55.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_46.export 2024-09-14T07:27:55.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:27:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_46.export 2024-09-14T07:27:56.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_46.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_46.export 2024-09-14T07:27:56.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:27:56.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:27:56.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_47 2024-09-14T07:27:56.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_47 snap 2024-09-14T07:27:56.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:27:56.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:56.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:27:56.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:27:56.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:27:56.602 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:27:56.600+0000 7f99ede58640 -1 reset not still connected to 0x559216789860 2024-09-14T07:27:58.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:27:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2024-09-14T07:27:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:27:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:27:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:27:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:27:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:27:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:27:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2024-09-14T07:27:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:27:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:59.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:27:59.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:27:59.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:27:59.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:27:59.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:27:59.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:27:59.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:27:59.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:27:59.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:27:59.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2024-09-14T07:27:59.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:27:59.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:27:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:27:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:27:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:27:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:27:59.924 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:27:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2024-09-14T07:28:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:28:00.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:28:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:28:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:28:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:28:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:28:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:28:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_47 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_47 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:28:00.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:28:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_47 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_47 ']' 2024-09-14T07:28:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_47' 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:00.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:00.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:00.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2024-09-14T07:28:00.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:00.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:00.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:00.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:00.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:00.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:00.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:00.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2024-09-14T07:28:00.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_47 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '' 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:00.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:00.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:00.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:00.427 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:00.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:00.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:00.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:00.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:00.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:00.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:00.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2024-09-14T07:28:00.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:00.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:00.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:00.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=8835' '!=' '' 2024-09-14T07:28:00.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:28:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=8835' = '' 2024-09-14T07:28:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:28:00.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:28:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:28:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:00.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:28:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:28:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=8835 2024-09-14T07:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 8835 2024-09-14T07:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_47 2024-09-14T07:28:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:00.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_47 ']' 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_47' 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:00.757 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:00.758 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:00.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:00.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:00.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:00.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:00.760 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2024-09-14T07:28:00.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_47 2024-09-14T07:28:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '' 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:00.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:00.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:00.919 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:00.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:00.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2024-09-14T07:28:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:00.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:01.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:01.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:01.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:01.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:01.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:01.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=8835' '!=' 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:01.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:01.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:28:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_47 2024-09-14T07:28:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_47 ']' 2024-09-14T07:28:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_47' 2024-09-14T07:28:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2024-09-14T07:28:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:01.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:01.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:01.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:01.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:01.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:01.435 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2024-09-14T07:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2024-09-14T07:28:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_47 2024-09-14T07:28:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '' 2024-09-14T07:28:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:01.587 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:01.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:01.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:01.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:01.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:01.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:01.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:01.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:01.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:01.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:01.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=8835' '!=' 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:01.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_47 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_47 ']' 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_47' 2024-09-14T07:28:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:02.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:02.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:02.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:02.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:02.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:02.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:02.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:02.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2024-09-14T07:28:02.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_47 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '' 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:02.666 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:02.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:02.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:02.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:02.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:02.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2024-09-14T07:28:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2024-09-14T07:28:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:02.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:02.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:02.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:02.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:02.779 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:02.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=8835' '!=' 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:02.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:02.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_47 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_47 ']' 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_47' 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2024-09-14T07:28:04.387 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:04.388 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:04.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:04.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:04.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2024-09-14T07:28:04.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:04.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:04.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:04.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:04.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:04.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:04.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:04.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:04.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:04.394 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:04.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:04.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:04.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2024-09-14T07:28:04.543 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_47 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '' 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:04.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:04.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:04.547 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:04.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:04.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2024-09-14T07:28:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2024-09-14T07:28:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:04.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:04.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:04.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:04.664 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834], [object_number=9, tag_tid=1, entry_tid=8833], [object_number=8, tag_tid=1, entry_tid=8832]]], state=connected] 2024-09-14T07:28:04.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=8835' '!=' 'object_number=11, tag_tid=1, entry_tid=8835' 2024-09-14T07:28:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:04.669 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:28:06.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_47 2024-09-14T07:28:06.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:06.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_47 ']' 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_47' 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:06.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:06.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:06.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:06.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:06.678 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:06.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2024-09-14T07:28:06.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:06.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_47 2024-09-14T07:28:06.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:06.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '' 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:06.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:06.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:06.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:06.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:06.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:06.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2024-09-14T07:28:06.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:06.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2024-09-14T07:28:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:06.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:28:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_47.status 2024-09-14T07:28:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_47.status 2024-09-14T07:28:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:06.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:06.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2024-09-14T07:28:06.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=8835], [object_number=10, tag_tid=1, entry_tid=8834]]], state=connected] 2024-09-14T07:28:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.status 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:06.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:06.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:28:06.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:07.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2024-09-14T07:28:08.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_47': 2024-09-14T07:28:08.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:28:08.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:28:08.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:28:08.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11ed270d39bb 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11ed270d39bb 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:43 2024 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:43 2024 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:18:11 2024 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11ed270d39bb 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:28:08.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 8bdc6ad1-f6d3-48f6-96e0-5414b9ab97b6 2024-09-14T07:28:08.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_47 snap 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.export 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_47.export 2024-09-14T07:28:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_47.export 2024-09-14T07:28:08.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.export 2024-09-14T07:28:08.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:08.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_47.export 2024-09-14T07:28:08.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:08.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_47.export 2024-09-14T07:28:08.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_47.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_47.export 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_48 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_48 snap 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:08.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:08.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:08.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:28:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:10.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:11.013 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9995859150119096) > weight (0.25) 2024-09-14T07:28:11.013 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5143792779284176) > weight (0.25) 2024-09-14T07:28:11.013 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-14T07:28:11.014 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-14T07:28:11.014 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T07:28:11.015 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:28:11.014+0000 7f4a5663d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 92252) UID: 1000 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:28:11.909 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:28:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2024-09-14T07:28:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:28:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:28:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:28:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:28:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:28:11.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:11.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:28:11.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:11.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:11.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:11.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2024-09-14T07:28:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T07:28:12.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:28:12.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2024-09-14T07:28:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:28:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:28:12.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:28:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:12.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:12.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:12.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:12.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2024-09-14T07:28:12.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:28:12.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:28:12.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:28:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:28:12.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_48 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_48 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_48 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_48 ']' 2024-09-14T07:28:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_48' 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:12.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:12.412 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:12.440 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:12.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:12.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:12.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:12.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T07:28:12.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:12.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:12.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:12.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2024-09-14T07:28:12.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:28:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:12.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:12.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:28:12.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:12.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:12.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:12.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:12.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:12.582 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:12.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:12.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:12.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_48 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '' 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:12.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:12.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:12.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:12.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:12.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:12.764 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498], [object_number=9, tag_tid=1, entry_tid=11497], [object_number=8, tag_tid=1, entry_tid=11496]]], state=connected] 2024-09-14T07:28:12.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2024-09-14T07:28:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:12.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:12.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:12.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:12.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:12.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:12.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:12.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:12.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498], [object_number=9, tag_tid=1, entry_tid=11497], [object_number=8, tag_tid=1, entry_tid=11496]]], state=connected] 2024-09-14T07:28:12.823 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:12.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:12.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:12.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=11499' '!=' '' 2024-09-14T07:28:12.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:28:12.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=11499' = '' 2024-09-14T07:28:12.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:12.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:12.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:28:12.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:12.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:28:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:28:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:28:12.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:12.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:28:12.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:12.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:28:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:28:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:12.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=11499 2024-09-14T07:28:12.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 11499 2024-09-14T07:28:12.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:12.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:12.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_48 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_48 ']' 2024-09-14T07:28:13.034 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_48' 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:13.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:13.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:13.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:13.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:13.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:13.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T07:28:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2024-09-14T07:28:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:28:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:13.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:13.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:13.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:13.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:13.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:13.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_48 2024-09-14T07:28:13.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '' 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:13.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:13.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:13.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:13.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498], [object_number=9, tag_tid=1, entry_tid=11497], [object_number=8, tag_tid=1, entry_tid=11496]]], state=connected] 2024-09-14T07:28:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:13.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:13.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:13.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:13.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:13.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:13.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:13.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:13.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:13.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:13.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498], [object_number=9, tag_tid=1, entry_tid=11497], [object_number=8, tag_tid=1, entry_tid=11496]]], state=connected] 2024-09-14T07:28:13.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:13.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:13.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:13.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=11499' '!=' 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:13.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:13.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_48 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_48 ']' 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_48' 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2024-09-14T07:28:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:13.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2024-09-14T07:28:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:13.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:13.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:13.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:13.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:13.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:13.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:13.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:13.885 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T07:28:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:28:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:14.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:14.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_48 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '' 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498], [object_number=9, tag_tid=1, entry_tid=11497], [object_number=8, tag_tid=1, entry_tid=11496]]], state=connected] 2024-09-14T07:28:14.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:14.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:14.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:14.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498], [object_number=9, tag_tid=1, entry_tid=11497], [object_number=8, tag_tid=1, entry_tid=11496]]], state=connected] 2024-09-14T07:28:14.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:14.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:14.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:14.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=11, tag_tid=1, entry_tid=11499' '!=' 'object_number=11, tag_tid=1, entry_tid=11499' 2024-09-14T07:28:14.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:14.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:28:15.015 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-14T07:28:15.015 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:28:15.016 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-14T07:28:15.016 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-14T07:28:15.016 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-14T07:28:15.019 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-14T07:28:15.019 INFO:tasks.rbd_mirror_thrash:waiting for 80 secs before thrashing 2024-09-14T07:28:15.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_48 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_48 ']' 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_48' 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2024-09-14T07:28:15.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:15.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:15.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:15.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:15.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:15.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:15.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:15.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:15.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:15.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:15.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:15.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:15.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:15.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:15.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:15.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:15.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:15.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-09-14T07:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:15.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:15.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:15.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:15.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:15.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:15.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:15.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2024-09-14T07:28:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_48 2024-09-14T07:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '' 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:15.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:15.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:15.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:15.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:15.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:15.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:15.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:15.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:15.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:15.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2024-09-14T07:28:15.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:15.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_48.status 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_48.status 2024-09-14T07:28:15.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:15.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:15.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2024-09-14T07:28:15.411 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:15.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 2 2024-09-14T07:28:15.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 2 2024-09-14T07:28:15.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:15.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:15.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=9, tag_tid=2, entry_tid=1], [object_number=8, tag_tid=2, entry_tid=0], [object_number=11, tag_tid=1, entry_tid=11499], [object_number=10, tag_tid=1, entry_tid=11498]]], state=connected] 2024-09-14T07:28:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.status 2024-09-14T07:28:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=9, tag_tid=2, entry_tid=1' -a 'object_number=9, tag_tid=2, entry_tid=1' = 'object_number=9, tag_tid=2, entry_tid=1' 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:28:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:16.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_48': 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 125e88a58a4b 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.125e88a58a4b 2024-09-14T07:28:16.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:16:53 2024 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:16:53 2024 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:54 2024 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 125e88a58a4b 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: ace9bd35-7f3f-43a1-ac1f-213c64b8a38a 2024-09-14T07:28:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_48 snap 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:28:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.export 2024-09-14T07:28:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_48.export 2024-09-14T07:28:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_48.export 2024-09-14T07:28:16.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.export 2024-09-14T07:28:16.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:16.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_48.export 2024-09-14T07:28:17.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:17.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_48.export 2024-09-14T07:28:17.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_48.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_48.export 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_49 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_49 snap 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:17.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:28:18.943 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:28:18.944 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:18.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:18.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2024-09-14T07:28:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:28:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:28:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:19.947 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:28:19.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:19.948 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:28:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-14T07:28:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:28:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-14T07:28:20.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:28:20.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:28:20.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:28:20.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:28:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:28:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:28:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_49 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_49 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:28:20.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:28:20.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:20.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:20.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_49 2024-09-14T07:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_49 ']' 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_49' 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:20.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:20.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:20.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:20.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:20.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:20.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:20.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:20.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:20.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2024-09-14T07:28:20.472 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:20.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_49 2024-09-14T07:28:20.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:20.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:20.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:20.473 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '' 2024-09-14T07:28:20.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:20.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:20.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:20.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:20.474 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:20.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:20.475 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:20.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2024-09-14T07:28:20.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:20.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:20.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:20.536 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:20.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:20.586 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:20.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=1' -a 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=2, tag_tid=1, entry_tid=3726' '!=' '' 2024-09-14T07:28:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ break 2024-09-14T07:28:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=2, tag_tid=1, entry_tid=3726' = '' 2024-09-14T07:28:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ last_mirror_pos='object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:28:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_tag=2 2024-09-14T07:28:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2024-09-14T07:28:20.593 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:20.594 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_tag=1 2024-09-14T07:28:20.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:20.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:28:20.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:20.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_entry=1 2024-09-14T07:28:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2024-09-14T07:28:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut -d= -f 2 2024-09-14T07:28:20.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_entry=3726 2024-09-14T07:28:20.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 2 = 1 -a 1 -le 3726 2024-09-14T07:28:20.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:20.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:20.598 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:20.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_49 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_49 ']' 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_49' 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:20.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:20.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:20.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:20.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:20.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:20.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:20.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:20.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:20.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:20.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:20.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:20.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2024-09-14T07:28:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_49 2024-09-14T07:28:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '' 2024-09-14T07:28:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:20.966 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=1' -a 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=2, tag_tid=1, entry_tid=3726' '!=' 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:21.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.4 2024-09-14T07:28:21.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_49 2024-09-14T07:28:21.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:21.477 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_49 ']' 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_49' 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:21.478 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:21.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:21.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:21.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:21.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:21.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:21.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2024-09-14T07:28:21.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_49 2024-09-14T07:28:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '' 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:21.637 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:21.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.638 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.690 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:21.692 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:21.693 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=1' -a 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=2, tag_tid=1, entry_tid=3726' '!=' 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:21.747 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.8 2024-09-14T07:28:22.548 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_49 2024-09-14T07:28:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_49 ']' 2024-09-14T07:28:22.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_49' 2024-09-14T07:28:22.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2024-09-14T07:28:22.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:22.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:22.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:22.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:22.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:22.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:22.551 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2024-09-14T07:28:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:22.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:22.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:22.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:22.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:22.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:22.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:22.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:22.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:22.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:22.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:22.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:22.562 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:22.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:22.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:22.563 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:22.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2024-09-14T07:28:22.717 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_49 2024-09-14T07:28:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:22.718 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '' 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:22.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:22.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:22.722 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:22.723 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:22.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:22.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:22.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:22.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:22.778 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:22.781 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2024-09-14T07:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:22.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:22.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:22.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:22.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:22.839 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:22.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:22.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:22.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=1' -a 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=2, tag_tid=1, entry_tid=3726' '!=' 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1.6 2024-09-14T07:28:24.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_49 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_49 ']' 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_49' 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:24.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:24.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:24.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:24.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:24.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2024-09-14T07:28:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:24.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:24.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:24.453 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:24.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:24.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:24.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:24.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:24.479 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2024-09-14T07:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_49 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '' 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:24.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:24.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:24.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:24.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:24.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:24.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:24.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:24.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:24.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:24.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:24.714 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:24.770 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=2, tag_tid=1, entry_tid=3726], [object_number=1, tag_tid=1, entry_tid=3725], [object_number=0, tag_tid=1, entry_tid=3724], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:24.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=1' -a 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:24.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 'object_number=2, tag_tid=1, entry_tid=3726' '!=' 'object_number=2, tag_tid=1, entry_tid=3726' 2024-09-14T07:28:24.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:24.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 2 2024-09-14T07:28:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_49 2024-09-14T07:28:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:26.802 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_49 ']' 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_49' 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2024-09-14T07:28:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:26.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:26.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:26.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:26.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:26.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:26.832 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:26.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:26.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:26.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2024-09-14T07:28:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_49 2024-09-14T07:28:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '' 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:26.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:26.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:26.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:27.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:27.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:27.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:27.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:27.033 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:27.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2024-09-14T07:28:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_49.status 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_49.status 2024-09-14T07:28:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:27.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:27.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2024-09-14T07:28:27.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 0 2024-09-14T07:28:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 0 2024-09-14T07:28:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:27.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=2, tag_tid=1, entry_tid=3726], [object_number=3, tag_tid=1, entry_tid=3723]]], state=connected] 2024-09-14T07:28:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.status 2024-09-14T07:28:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=1' -a 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=1, tag_tid=2, entry_tid=1' 2024-09-14T07:28:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2024-09-14T07:28:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_49': 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 1182f1940a8f 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.1182f1940a8f 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:28:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:17:05 2024 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:17:05 2024 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:17:05 2024 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 1182f1940a8f 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 21c98667-c0b3-4419-8937-d3a13342685e 2024-09-14T07:28:28.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:28:28.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:28.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_49 snap 2024-09-14T07:28:28.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:28.172 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:28:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:28:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.export 2024-09-14T07:28:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_49.export 2024-09-14T07:28:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_49.export 2024-09-14T07:28:28.173 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.export 2024-09-14T07:28:28.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:28.358 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_49.export 2024-09-14T07:28:28.592 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:28.601 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_49.export 2024-09-14T07:28:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_49.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_49.export 2024-09-14T07:28:28.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:28.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:28.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_50 2024-09-14T07:28:28.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_snap cluster2 mirror image_50 snap 2024-09-14T07:28:28.711 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:28.712 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2024-09-14T07:28:29.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2024-09-14T07:28:29.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:29.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:29.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:29.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2024-09-14T07:28:29.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=started 2024-09-14T07:28:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2024-09-14T07:28:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=started 2024-09-14T07:28:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local status_result 2024-09-14T07:28:30.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=stopped 2024-09-14T07:28:30.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2024-09-14T07:28:30.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_instance=cluster1 2024-09-14T07:28:30.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:30.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance=cluster1 2024-09-14T07:28:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local loop_instance 2024-09-14T07:28:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sleep 0 2024-09-14T07:28:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -i state 2024-09-14T07:28:30.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:30.876 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ seq 0 3 2024-09-14T07:28:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:0 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:0 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:0 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=0 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=0 2024-09-14T07:28:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:30.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:30.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:30.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:30.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2024-09-14T07:28:31.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T07:28:31.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:28:31.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:1 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:31.018 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:31.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:31.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:31.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:1 2024-09-14T07:28:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=1 2024-09-14T07:28:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:31.043 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2024-09-14T07:28:31.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T07:28:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:28:31.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:31.160 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:2 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:28:31.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:31.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:31.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:28:31.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:2 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:2 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:31.163 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=2 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=2 2024-09-14T07:28:31.164 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:28:31.188 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:28:31.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:28:31.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:28:31.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2024-09-14T07:28:31.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T07:28:31.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror status : get status for rbd mirror 2024-09-14T07:28:31.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:31.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_50 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1:3 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ shift 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=cluster 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval cluster=cluster1 2024-09-14T07:28:31.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:31.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:28:31.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:28:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:28:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local local_cluster=cluster1:3 2024-09-14T07:28:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster=cluster1 2024-09-14T07:28:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance 2024-09-14T07:28:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:28:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local val=cluster1:3 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local cluster_var_name=local_cluster 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local instance_var_name=instance 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ cluster=cluster1 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval local_cluster=cluster1 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ local_cluster=cluster1 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ eval instance=3 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ instance=3 2024-09-14T07:28:31.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:28:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:28:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:28:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:28:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_50 2024-09-14T07:28:31.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ return 0 2024-09-14T07:28:31.444 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ status_result=' "state": "Replaying"' 2024-09-14T07:28:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo ' "state": "Replaying"' 2024-09-14T07:28:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ grep -i Replaying 2024-09-14T07:28:31.446 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: "state": "Replaying" 2024-09-14T07:28:31.447 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=started 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test started = started 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_50 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' journal = journal ']' 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror image_50 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local local_cluster=cluster1 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s master_pos mirror_pos last_mirror_pos 2024-09-14T07:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2024-09-14T07:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-14T07:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.2 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ flush cluster1 mirror image_50 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local 'cmd=rbd mirror flush' 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n image_50 ']' 2024-09-14T07:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmd='rbd mirror flush mirror/image_50' 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_instance=cluster1 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance=cluster1 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loop_instance 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:31.650 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 '!=' cluster1 ']' 2024-09-14T07:28:31.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:28:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2024-09-14T07:28:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:28:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:0 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:28:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:0 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:0 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-14T07:28:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=0 2024-09-14T07:28:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=0 2024-09-14T07:28:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-14T07:28:31.679 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-14T07:28:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2024-09-14T07:28:31.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T07:28:31.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:31.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:31.794 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:1 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=1 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:31.795 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:31.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=1 2024-09-14T07:28:31.796 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:1 cluster1 2024-09-14T07:28:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:1 2024-09-14T07:28:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-14T07:28:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=1 2024-09-14T07:28:31.798 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-14T07:28:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-14T07:28:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2024-09-14T07:28:31.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T07:28:31.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:31.933 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_50 2024-09-14T07:28:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:28:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:2 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=2 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=2 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:2 cluster1 2024-09-14T07:28:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:2 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:2 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=2 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=2 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=2 2024-09-14T07:28:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-14T07:28:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:28:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:28:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-14T07:28:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_50 2024-09-14T07:28:32.024 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8651511752854537) > weight (0.25) 2024-09-14T07:28:32.024 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8654894758929581) > weight (0.25) 2024-09-14T07:28:32.024 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:28:32.025 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:28:32.025 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-14T07:28:32.025 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:28:32.025 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:28:32.026 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:28:32.026+0000 7f1c7a086640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 104313) UID: 1000 2024-09-14T07:28:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-14T07:28:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd mirror flush : flush rbd mirror 2024-09-14T07:28:32.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:admin_socket: invalid command 2024-09-14T07:28:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_50 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1:3 2024-09-14T07:28:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=3 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=3 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ daemon_asok_file cluster1:3 cluster1 2024-09-14T07:28:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local local_cluster=cluster1:3 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local val=cluster1:3 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster_var_name=local_cluster 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local instance_var_name=instance 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:28:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=3 2024-09-14T07:28:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-14T07:28:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval local_cluster=cluster1 2024-09-14T07:28:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ local_cluster=cluster1 2024-09-14T07:28:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ eval instance=3 2024-09-14T07:28:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ instance=3 2024-09-14T07:28:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-14T07:28:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:28:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:28:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-14T07:28:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_50 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_master_journal_position cluster2 mirror image_50 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_50 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_50 '' 2024-09-14T07:28:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_50 2024-09-14T07:28:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local id_regexp= 2024-09-14T07:28:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_50.status 2024-09-14T07:28:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_50.status 2024-09-14T07:28:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:32.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.status 2024-09-14T07:28:32.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2024-09-14T07:28:32.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.status 2024-09-14T07:28:32.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:28:32.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:28:32.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:32.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:28:32.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:28:32.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.status 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ master_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_50 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_50 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local 'id_regexp=..*' 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ mkfname cluster2-mirror-image_50.status 2024-09-14T07:28:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ echo cluster2-mirror-image_50.status 2024-09-14T07:28:32.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+++ sed -e 's|[/ ]|_|g' 2024-09-14T07:28:32.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local status_log=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.status 2024-09-14T07:28:32.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2024-09-14T07:28:32.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ tee /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.status 2024-09-14T07:28:32.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:minimum_set: 1 2024-09-14T07:28:32.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:active_set: 1 2024-09-14T07:28:32.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:registered clients: 2024-09-14T07:28:32.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:28:32.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: [id=d689b02d-8125-4859-9c44-bebfa1249cbd, commit_position=[positions=[[object_number=5, tag_tid=3, entry_tid=1], [object_number=4, tag_tid=3, entry_tid=0], [object_number=7, tag_tid=1, entry_tid=6611], [object_number=6, tag_tid=1, entry_tid=6610]]], state=connected] 2024-09-14T07:28:32.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.status 2024-09-14T07:28:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ mirror_pos='object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:28:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 'object_number=5, tag_tid=3, entry_tid=1' -a 'object_number=5, tag_tid=3, entry_tid=1' = 'object_number=5, tag_tid=3, entry_tid=1' 2024-09-14T07:28:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-14T07:28:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:28:33.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2024-09-14T07:28:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_50': 2024-09-14T07:28:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 128 MiB in 32 objects 2024-09-14T07:28:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:28:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 1 2024-09-14T07:28:33.418 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 11b0ce6b23b1 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.11b0ce6b23b1 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:17:11 2024 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:17:11 2024 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:26:53 2024 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: protected: False 2024-09-14T07:28:33.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 11b0ce6b23b1 2024-09-14T07:28:33.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:28:33.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:28:33.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 63c77e78-173a-4afa-a969-253448172b27 2024-09-14T07:28:33.420 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: false 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ compare_image_snaps mirror image_50 snap 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap_name=snap 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local ret=0 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local rmt_export=/tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.export 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local loc_export=/tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_50.export 2024-09-14T07:28:33.423 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_50.export 2024-09-14T07:28:33.424 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.export 2024-09-14T07:28:33.580 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:33.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_50.export 2024-09-14T07:28:33.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-14T07:28:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cmp /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_50.export 2024-09-14T07:28:33.934 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -f /tmp/tmp.gBxNEDbwzY/cluster2-mirror-image_50.export /tmp/tmp.gBxNEDbwzY/cluster1-mirror-image_50.export 2024-09-14T07:28:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: delete many images' 2024-09-14T07:28:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T07:28:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T07:28:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 07:28:33 TEST: delete many images 2024-09-14T07:28:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 07:28:33 TEST: delete many images 2024-09-14T07:28:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 50 2024-09-14T07:28:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_1 2024-09-14T07:28:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2024-09-14T07:28:33.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:33.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:33.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:28:33.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:33.980 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2024-09-14T07:28:34.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:34.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_1 2024-09-14T07:28:34.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:34.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:34.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:28:34.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:34.913 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:34.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_1 2024-09-14T07:28:34.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:34.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:34.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:28:34.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-09-14T07:28:34.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-09-14T07:28:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_2 2024-09-14T07:28:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2024-09-14T07:28:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:28:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2024-09-14T07:28:35.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_2 2024-09-14T07:28:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:28:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:35.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T07:28:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:28:35.923 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T07:28:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T07:28:36.041 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:28:36.038+0000 7f6aa3428640 -1 rbd::mirror::ImageMap: 0x55a2b303bb00 handle_peer_ack_remove: failed to remove global_image_id=6f654538-f738-4f37-ad20-6fa40855e451 2024-09-14T07:28:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:36.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:36.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:36.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_3 2024-09-14T07:28:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2024-09-14T07:28:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:28:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:36.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2024-09-14T07:28:36.922 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_3 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:28:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T07:28:37.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T07:28:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_4 2024-09-14T07:28:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2024-09-14T07:28:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:37.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:28:37.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:37.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2024-09-14T07:28:37.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:37.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_4 2024-09-14T07:28:37.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:37.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:37.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:28:37.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:37.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:37.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T07:28:37.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:37.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:37.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:28:37.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T07:28:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T07:28:38.127 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:28:38.127 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:28:38.128 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:28:38.132 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:28:38.132 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-09-14T07:28:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:38.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:38.229 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_5 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:38.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2024-09-14T07:28:38.303 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:28:38.302+0000 7f7dd9467640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 1 2024-09-14T07:28:38.926 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:38.954 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_5 2024-09-14T07:28:38.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:38.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:38.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:28:38.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:38.955 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:38.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T07:28:38.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:38.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:38.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:28:38.956 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T07:28:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T07:28:39.081 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:28:39.078+0000 7f6aa3428640 -1 rbd::mirror::ImageMap: 0x55a2b303bb00 handle_peer_ack_remove: failed to remove global_image_id=3bb9cf91-5d7a-40a8-a4f1-910b7a05cc33 2024-09-14T07:28:39.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_6 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:39.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2024-09-14T07:28:39.942 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_6 2024-09-14T07:28:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:28:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T07:28:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:28:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T07:28:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T07:28:40.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:40.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_7 2024-09-14T07:28:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2024-09-14T07:28:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:28:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2024-09-14T07:28:40.953 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:40.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_7 2024-09-14T07:28:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:28:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T07:28:40.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:40.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:40.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:28:40.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T07:28:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T07:28:41.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:41.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:41.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_8 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2024-09-14T07:28:41.282 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:28:41.278+0000 7fe057a2c640 -1 rbd::mirror::ImageMap: 0x55cecc9537a0 handle_peer_ack_remove: failed to remove global_image_id=306faee7-90b2-4c72-9856-ceb4a60d5ce4 2024-09-14T07:28:41.755 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.751+0000 7fea5efdb640 -1 reset not still connected to 0x558445208750 2024-09-14T07:28:41.758 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.755+0000 7fea5efdb640 -1 reset not still connected to 0x55844520aa90 2024-09-14T07:28:41.758 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.755+0000 7fea5efdb640 -1 reset not still connected to 0x558445d50f70 2024-09-14T07:28:41.762 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.755+0000 7fea5efdb640 -1 reset not still connected to 0x558446184b60 2024-09-14T07:28:41.764 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.759+0000 7fea5efdb640 -1 reset not still connected to 0x55845a7948f0 2024-09-14T07:28:41.764 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.759+0000 7fea5efdb640 -1 reset not still connected to 0x55846479d520 2024-09-14T07:28:41.765 INFO:tasks.ceph.cluster1.osd.0.smithi129.stderr:2024-09-14T07:28:41.759+0000 7fea5efdb640 -1 reset not still connected to 0x558471f1a000 2024-09-14T07:28:41.925 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:41.939 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:41.938+0000 7fb3f8d94640 -1 reset not still connected to 0x5614c0cdad00 2024-09-14T07:28:41.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_8 2024-09-14T07:28:41.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:41.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:41.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:28:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T07:28:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:28:41.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T07:28:41.994 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:28:41.994+0000 7fe05722b640 -1 rbd::mirror::ImageMap: 0x55cecc9537a0 handle_peer_ack_remove: failed to remove global_image_id=3bb9cf91-5d7a-40a8-a4f1-910b7a05cc33 2024-09-14T07:28:42.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T07:28:42.226 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:42.230 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_9 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:42.231 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2024-09-14T07:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:43.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_9 2024-09-14T07:28:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:28:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T07:28:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:28:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T07:28:43.094 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T07:28:43.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_10 2024-09-14T07:28:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2024-09-14T07:28:43.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:43.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:43.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:28:43.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:43.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2024-09-14T07:28:43.357 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:28:43.354+0000 7fe05722b640 -1 rbd::mirror::ImageMap: 0x55cecc9537a0 handle_peer_ack_remove: failed to remove global_image_id=cee00132-8847-4ad7-ad34-a0946114ca65 2024-09-14T07:28:43.381 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:43.378+0000 7fb3e694f640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:28:43.385 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:43.382+0000 7fb3e7150640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:28:43.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_10 2024-09-14T07:28:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:28:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T07:28:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:28:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T07:28:43.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T07:28:44.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:44.121 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_11 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2024-09-14T07:28:44.189 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:44.186+0000 7fb3e7150640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 2 2024-09-14T07:28:44.195 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:44.194+0000 7fb3ea957640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 2 2024-09-14T07:28:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_11 2024-09-14T07:28:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:28:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_11 2024-09-14T07:28:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:28:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2024-09-14T07:28:45.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2024-09-14T07:28:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:45.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:45.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:45.267 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_12 2024-09-14T07:28:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2024-09-14T07:28:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:28:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:45.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2024-09-14T07:28:45.333 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:45.330+0000 7fb3e414a640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 3 2024-09-14T07:28:45.334 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:28:45.334+0000 7fb3e614e640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:28:45.982 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:46.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_12 2024-09-14T07:28:46.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:28:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_12 2024-09-14T07:28:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:28:46.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2024-09-14T07:28:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2024-09-14T07:28:46.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_13 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:46.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2024-09-14T07:28:46.975 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:46.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_13 2024-09-14T07:28:46.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:47.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:47.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:28:47.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:47.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:47.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_13 2024-09-14T07:28:47.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:47.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:47.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:28:47.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2024-09-14T07:28:47.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2024-09-14T07:28:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:47.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:47.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:47.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_14 2024-09-14T07:28:47.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2024-09-14T07:28:47.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:47.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:47.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:28:47.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:47.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2024-09-14T07:28:47.322 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:28:47.322+0000 7fe057a2c640 -1 rbd::mirror::ImageMap: 0x55cecc9537a0 handle_peer_ack_remove: failed to remove global_image_id=9f051053-923e-43cd-8832-0a8506372e9f 2024-09-14T07:28:48.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_14 2024-09-14T07:28:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:28:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_14 2024-09-14T07:28:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:48.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:28:48.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2024-09-14T07:28:48.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2024-09-14T07:28:48.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_15 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:28:48.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:48.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2024-09-14T07:28:48.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_15 2024-09-14T07:28:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:28:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:48.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_15 2024-09-14T07:28:48.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:48.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:48.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:28:48.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2024-09-14T07:28:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2024-09-14T07:28:49.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_16 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:49.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2024-09-14T07:28:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:50.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_16 2024-09-14T07:28:50.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:28:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_16 2024-09-14T07:28:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:50.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:50.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:28:50.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2024-09-14T07:28:50.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2024-09-14T07:28:50.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_17 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:28:50.324 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:50.325 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2024-09-14T07:28:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_17 2024-09-14T07:28:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:28:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_17 2024-09-14T07:28:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:28:51.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2024-09-14T07:28:51.075 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2024-09-14T07:28:51.288 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_18 2024-09-14T07:28:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2024-09-14T07:28:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:28:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2024-09-14T07:28:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:52.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_18 2024-09-14T07:28:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:28:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_18 2024-09-14T07:28:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:28:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2024-09-14T07:28:52.138 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2024-09-14T07:28:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_19 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:52.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2024-09-14T07:28:52.395 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:28:52.394+0000 7fe05722b640 -1 rbd::mirror::ImageMap: 0x55cecc9537a0 handle_peer_ack_remove: failed to remove global_image_id=3a6a361c-afda-4451-826d-eeb995f6a4df 2024-09-14T07:28:53.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_19 2024-09-14T07:28:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:28:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:53.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:53.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_19 2024-09-14T07:28:53.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:53.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:53.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:28:53.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2024-09-14T07:28:53.332 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2024-09-14T07:28:53.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_20 2024-09-14T07:28:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2024-09-14T07:28:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:53.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:53.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:28:53.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:53.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2024-09-14T07:28:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_20 2024-09-14T07:28:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:28:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:53.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_20 2024-09-14T07:28:53.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:53.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:53.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:28:53.862 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2024-09-14T07:28:53.911 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2024-09-14T07:28:54.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_21 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2024-09-14T07:28:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_21 2024-09-14T07:28:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:28:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:55.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_21 2024-09-14T07:28:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:28:55.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2024-09-14T07:28:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2024-09-14T07:28:55.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_22 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:55.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2024-09-14T07:28:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_22 2024-09-14T07:28:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:28:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_22 2024-09-14T07:28:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:28:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2024-09-14T07:28:56.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2024-09-14T07:28:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_23 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:28:56.588 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:56.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2024-09-14T07:28:57.037 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_23 2024-09-14T07:28:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:28:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_23 2024-09-14T07:28:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:28:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2024-09-14T07:28:57.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2024-09-14T07:28:57.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_24 2024-09-14T07:28:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2024-09-14T07:28:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:28:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2024-09-14T07:28:58.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_24 2024-09-14T07:28:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:28:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_24 2024-09-14T07:28:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:28:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2024-09-14T07:28:58.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2024-09-14T07:28:58.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_25 2024-09-14T07:28:58.383 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2024-09-14T07:28:58.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:58.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:58.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:28:58.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:58.384 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2024-09-14T07:28:58.445 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:28:58.442+0000 7f7dd9467640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 2 2024-09-14T07:28:58.447 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:28:58.446+0000 7f7ddd46f640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 4 2024-09-14T07:28:58.452 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:28:58.446+0000 7f7dd8465640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 5 2024-09-14T07:28:58.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:28:58.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_25 2024-09-14T07:28:58.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:58.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:58.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:28:58.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:28:58.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:28:58.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_25 2024-09-14T07:28:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:28:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2024-09-14T07:28:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2024-09-14T07:28:59.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:28:59.078+0000 7fe057a2c640 -1 rbd::mirror::ImageMap: 0x55cecc9537a0 handle_peer_ack_remove: failed to remove global_image_id=4b109641-f23e-48e0-9065-a1f448eaba74 2024-09-14T07:28:59.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:28:59.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:28:59.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:28:59.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_26 2024-09-14T07:28:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2024-09-14T07:28:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:28:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:28:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:28:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:28:59.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2024-09-14T07:29:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_26 2024-09-14T07:29:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:29:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:00.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_26 2024-09-14T07:29:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:29:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2024-09-14T07:29:00.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2024-09-14T07:29:00.373 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_27 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:00.378 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2024-09-14T07:29:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:01.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_27 2024-09-14T07:29:01.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:01.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:01.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:29:01.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:01.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_27 2024-09-14T07:29:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:29:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2024-09-14T07:29:01.185 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2024-09-14T07:29:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_28 2024-09-14T07:29:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2024-09-14T07:29:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:29:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2024-09-14T07:29:02.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_28 2024-09-14T07:29:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:29:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:02.131 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:02.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_28 2024-09-14T07:29:02.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:02.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:02.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:29:02.132 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2024-09-14T07:29:02.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2024-09-14T07:29:02.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_29 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:02.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2024-09-14T07:29:03.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_29 2024-09-14T07:29:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:29:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:03.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_29 2024-09-14T07:29:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:29:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2024-09-14T07:29:03.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2024-09-14T07:29:03.393 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_30 2024-09-14T07:29:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2024-09-14T07:29:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:03.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:03.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:29:03.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:03.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2024-09-14T07:29:03.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_30 2024-09-14T07:29:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:29:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_30 2024-09-14T07:29:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:29:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2024-09-14T07:29:03.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2024-09-14T07:29:04.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_31 2024-09-14T07:29:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2024-09-14T07:29:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:29:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2024-09-14T07:29:04.196 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:04.193+0000 7fb3eb959640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 1 2024-09-14T07:29:04.198 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:04.197+0000 7fb3e514c640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:29:05.130 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:05.155 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_31 2024-09-14T07:29:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:29:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:05.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_31 2024-09-14T07:29:05.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:05.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:05.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:29:05.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2024-09-14T07:29:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2024-09-14T07:29:05.433 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_32 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2024-09-14T07:29:06.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_32 2024-09-14T07:29:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:29:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_32 2024-09-14T07:29:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:29:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2024-09-14T07:29:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2024-09-14T07:29:06.400 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:06.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:06.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:06.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_33 2024-09-14T07:29:06.404 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2024-09-14T07:29:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:29:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2024-09-14T07:29:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_33 2024-09-14T07:29:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:29:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_33 2024-09-14T07:29:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:29:07.162 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2024-09-14T07:29:07.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2024-09-14T07:29:07.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_34 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:29:07.431 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:07.432 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2024-09-14T07:29:08.151 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:08.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_34 2024-09-14T07:29:08.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:08.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:08.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:29:08.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_34 2024-09-14T07:29:08.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:08.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:08.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:29:08.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2024-09-14T07:29:08.232 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2024-09-14T07:29:08.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_35 2024-09-14T07:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2024-09-14T07:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:08.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:29:08.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:08.442 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2024-09-14T07:29:08.846 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_35 2024-09-14T07:29:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:29:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_35 2024-09-14T07:29:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:29:08.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2024-09-14T07:29:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2024-09-14T07:29:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_36 2024-09-14T07:29:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2024-09-14T07:29:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:09.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:29:09.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:09.146 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2024-09-14T07:29:10.171 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:10.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_36 2024-09-14T07:29:10.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:10.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:10.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:29:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_36 2024-09-14T07:29:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:29:10.209 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2024-09-14T07:29:10.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2024-09-14T07:29:10.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_37 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2024-09-14T07:29:10.532 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:29:10.529+0000 7f7ddb46b640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 5 2024-09-14T07:29:10.532 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:29:10.529+0000 7f7ddbc6c640 -1 bdev(0x55ca4ea80c00 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 4 2024-09-14T07:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:11.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_37 2024-09-14T07:29:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:29:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_37 2024-09-14T07:29:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:29:11.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2024-09-14T07:29:11.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2024-09-14T07:29:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:11.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:11.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:11.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_38 2024-09-14T07:29:11.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2024-09-14T07:29:11.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:29:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2024-09-14T07:29:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:12.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_38 2024-09-14T07:29:12.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:12.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:12.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:29:12.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:12.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:12.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_38 2024-09-14T07:29:12.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:12.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:12.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:29:12.211 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2024-09-14T07:29:12.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2024-09-14T07:29:12.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_39 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:29:12.487 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:12.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2024-09-14T07:29:13.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:13.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_39 2024-09-14T07:29:13.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:13.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:13.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:29:13.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:13.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:13.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_39 2024-09-14T07:29:13.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:13.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:13.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:29:13.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2024-09-14T07:29:13.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2024-09-14T07:29:13.480 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:13.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:13.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:13.484 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_40 2024-09-14T07:29:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2024-09-14T07:29:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:29:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2024-09-14T07:29:13.556 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:13.553+0000 7fb3e614e640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:29:13.557 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:13.553+0000 7fb3eb158640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:29:13.620 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:29:13.606+0000 7f99ede58640 -1 reset not still connected to 0x559210517110 2024-09-14T07:29:13.620 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:29:13.610+0000 7f99ede58640 -1 reset not still connected to 0x5592118ce1a0 2024-09-14T07:29:13.636 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:29:13.610+0000 7f99ede58640 -1 reset not still connected to 0x559211a221a0 2024-09-14T07:29:13.637 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:29:13.614+0000 7f99ede58640 -1 reset not still connected to 0x559214ba9ad0 2024-09-14T07:29:13.638 INFO:tasks.ceph.cluster1.osd.2.smithi129.stderr:2024-09-14T07:29:13.626+0000 7f99ede58640 -1 reset not still connected to 0x559218b045b0 2024-09-14T07:29:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_40 2024-09-14T07:29:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:29:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_40 2024-09-14T07:29:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:29:13.855 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2024-09-14T07:29:13.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2024-09-14T07:29:14.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_41 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2024-09-14T07:29:15.187 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_41 2024-09-14T07:29:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:29:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_41 2024-09-14T07:29:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:29:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2024-09-14T07:29:15.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2024-09-14T07:29:15.488 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:15.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:15.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:15.492 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_42 2024-09-14T07:29:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2024-09-14T07:29:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:29:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2024-09-14T07:29:16.210 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:16.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_42 2024-09-14T07:29:16.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:16.236 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_42 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:29:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2024-09-14T07:29:16.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2024-09-14T07:29:16.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_43 2024-09-14T07:29:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2024-09-14T07:29:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:29:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2024-09-14T07:29:17.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_43 2024-09-14T07:29:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:29:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_43 2024-09-14T07:29:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:17.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:17.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:29:17.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2024-09-14T07:29:17.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2024-09-14T07:29:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_44 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2024-09-14T07:29:18.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_44 2024-09-14T07:29:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:29:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:18.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_44 2024-09-14T07:29:18.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:18.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:18.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:29:18.234 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2024-09-14T07:29:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2024-09-14T07:29:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:18.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:18.499 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_45 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2024-09-14T07:29:18.574 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:18.573+0000 7fb3e494b640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:29:18.575 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:18.573+0000 7fb3eb158640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:29:18.575 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:29:18.573+0000 7fb3ea156640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2024-09-14T07:29:18.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_45 2024-09-14T07:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_45 2024-09-14T07:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2024-09-14T07:29:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2024-09-14T07:29:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_46 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:29:19.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:19.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2024-09-14T07:29:20.248 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:20.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_46 2024-09-14T07:29:20.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:20.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:20.275 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:29:20.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:20.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:20.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_46 2024-09-14T07:29:20.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:20.276 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:20.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:29:20.277 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2024-09-14T07:29:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2024-09-14T07:29:20.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:20.549 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_47 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:20.550 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2024-09-14T07:29:21.227 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_47 2024-09-14T07:29:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:29:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_47 2024-09-14T07:29:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:29:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2024-09-14T07:29:21.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2024-09-14T07:29:21.521 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:21.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:21.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:21.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_48 2024-09-14T07:29:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2024-09-14T07:29:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:29:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:21.526 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2024-09-14T07:29:22.247 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_48 2024-09-14T07:29:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:29:22.270 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:22.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:22.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_48 2024-09-14T07:29:22.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:22.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:22.271 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:29:22.272 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2024-09-14T07:29:22.322 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2024-09-14T07:29:22.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_49 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2024-09-14T07:29:23.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:23.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_49 2024-09-14T07:29:23.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:23.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:23.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:29:23.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:23.289 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:23.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_49 2024-09-14T07:29:23.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:23.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:23.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:29:23.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2024-09-14T07:29:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2024-09-14T07:29:23.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_50 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local snap=snap 2024-09-14T07:29:23.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2024-09-14T07:29:23.791 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing snap: 100% complete...done. 2024-09-14T07:29:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_50 2024-09-14T07:29:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:29:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:29:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:29:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_50 2024-09-14T07:29:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:29:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2024-09-14T07:29:23.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2024-09-14T07:29:24.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-14T07:29:24.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:24.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: image deletions should propagate' 2024-09-14T07:29:24.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T07:29:24.108 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T07:29:24.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 07:29:24 TEST: image deletions should propagate 2024-09-14T07:29:24.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 07:29:24 TEST: image deletions should propagate 2024-09-14T07:29:24.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_pool_images cluster1 mirror 0 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_count=0 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local count 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local last_count=0 2024-09-14T07:29:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ true 2024-09-14T07:29:24.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 40 2024-09-14T07:29:24.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:29:24.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 1 -ne 1 2024-09-14T07:29:24.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T07:29:24.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'images: ' 2024-09-14T07:29:24.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 2 2024-09-14T07:29:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:29:24.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ count=0 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state 2024-09-14T07:29:24.158 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 40 2024-09-14T07:29:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in `seq 1 40` 2024-09-14T07:29:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 1 -ne 1 2024-09-14T07:29:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-14T07:29:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep 'image health:' 2024-09-14T07:29:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cut '-d ' -f 3 2024-09-14T07:29:24.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: failed to query services: (13) Permission denied 2024-09-14T07:29:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ state=OK 2024-09-14T07:29:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test OK = ERROR 2024-09-14T07:29:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test OK = OK 2024-09-14T07:29:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 50 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_1 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:24.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:29:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:29:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:29:24.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-14T07:29:24.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:24.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:24.244 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2024-09-14T07:29:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:29:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:29:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:29:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.390 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-14T07:29:24.395 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_2 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T07:29:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T07:29:24.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:24.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:24.438 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T07:29:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:29:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T07:29:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_3 2024-09-14T07:29:24.620 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:29:24.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T07:29:24.622 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T07:29:24.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-09-14T07:29:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:29:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T07:29:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:29:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T07:29:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.821 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_4 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-09-14T07:29:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T07:29:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T07:29:24.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:24.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-09-14T07:29:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:24.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:24.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:29:24.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:24.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:24.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:24.970 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T07:29:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:29:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T07:29:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.017 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_5 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-09-14T07:29:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T07:29:25.023 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T07:29:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:25.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:25.168 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.169 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:29:25.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.170 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T07:29:25.216 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_6 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:29:25.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:29:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T07:29:25.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:25.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:25.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:25.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-09-14T07:29:25.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:29:25.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:25.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:25.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:25.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:29:25.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:29:25.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:29:25.368 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.619 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_7 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T07:29:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T07:29:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:25.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T07:29:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:29:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T07:29:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T07:29:25.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_8 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T07:29:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:25.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T07:29:25.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:26.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-09-14T07:29:26.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:29:26.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:26.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:26.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:26.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T07:29:26.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:29:26.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T07:29:26.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T07:29:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_9 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T07:29:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:29:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T07:29:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T07:29:26.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:26.101 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:26.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-09-14T07:29:26.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:29:26.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:26.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:26.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:26.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T07:29:26.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:29:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T07:29:26.204 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T07:29:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_10 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T07:29:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T07:29:26.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:26.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:26.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-09-14T07:29:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:29:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T07:29:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:29:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T07:29:26.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T07:29:26.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:26.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:26.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:26.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:26.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_11 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_11 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-09-14T07:29:26.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-09-14T07:29:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:26.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:26.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:26.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2024-09-14T07:29:26.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_11 2024-09-14T07:29:26.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_11 2024-09-14T07:29:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_11 2024-09-14T07:29:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-09-14T07:29:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.651 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-09-14T07:29:26.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_12 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_12 2024-09-14T07:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-09-14T07:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-09-14T07:29:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:26.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_12 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:26.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_12 2024-09-14T07:29:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_12 2024-09-14T07:29:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-09-14T07:29:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.872 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-09-14T07:29:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_13 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_13 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:29:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-09-14T07:29:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-09-14T07:29:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2024-09-14T07:29:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_13 2024-09-14T07:29:27.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:27.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:27.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_13 2024-09-14T07:29:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_13 2024-09-14T07:29:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-09-14T07:29:27.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.066 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-09-14T07:29:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_14 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.072 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_14 2024-09-14T07:29:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:29:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-09-14T07:29:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-09-14T07:29:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:27.115 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2024-09-14T07:29:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_14 2024-09-14T07:29:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_14 2024-09-14T07:29:27.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_14 2024-09-14T07:29:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-09-14T07:29:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_15 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.268 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_15 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-09-14T07:29:27.269 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-09-14T07:29:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:27.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:27.413 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_15 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:27.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_15 2024-09-14T07:29:27.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_15 2024-09-14T07:29:27.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-09-14T07:29:27.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_16 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_16 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-09-14T07:29:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-09-14T07:29:27.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:27.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:27.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_16 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:27.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:27.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_16 2024-09-14T07:29:27.614 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_16 2024-09-14T07:29:27.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-09-14T07:29:27.615 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_17 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:29:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_17 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-09-14T07:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.700 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-09-14T07:29:27.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:27.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:27.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_17 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:27.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:27.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_17 2024-09-14T07:29:27.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_17 2024-09-14T07:29:27.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-09-14T07:29:27.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_18 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_18 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-09-14T07:29:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:27.899 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-09-14T07:29:27.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2024-09-14T07:29:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_18 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_18 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_18 2024-09-14T07:29:28.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-09-14T07:29:28.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-09-14T07:29:28.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:28.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:28.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:28.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:28.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_19 2024-09-14T07:29:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2024-09-14T07:29:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:29:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_19 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:29:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-09-14T07:29:28.063 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-09-14T07:29:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_19 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_19 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_19 2024-09-14T07:29:28.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-09-14T07:29:28.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-09-14T07:29:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_20 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_20 2024-09-14T07:29:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:29:28.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-09-14T07:29:28.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-09-14T07:29:28.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:28.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:28.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_20 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_20 2024-09-14T07:29:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_20 2024-09-14T07:29:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-09-14T07:29:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_21 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2024-09-14T07:29:28.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_21 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-09-14T07:29:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-09-14T07:29:28.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:28.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:28.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2024-09-14T07:29:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_21 2024-09-14T07:29:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:28.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_21 2024-09-14T07:29:28.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_21 2024-09-14T07:29:28.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-09-14T07:29:28.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.648 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_22 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_22 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-09-14T07:29:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-09-14T07:29:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2024-09-14T07:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_22 2024-09-14T07:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_22 2024-09-14T07:29:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_22 2024-09-14T07:29:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-09-14T07:29:28.801 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.847 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_23 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:29:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_23 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-09-14T07:29:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:28.891 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-09-14T07:29:28.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:28.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2024-09-14T07:29:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:28.997 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_23 2024-09-14T07:29:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:28.998 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_23 2024-09-14T07:29:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_23 2024-09-14T07:29:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-09-14T07:29:28.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-09-14T07:29:29.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:29.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_24 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_24 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-09-14T07:29:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-09-14T07:29:29.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:29.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:29.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_24 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_24 2024-09-14T07:29:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_24 2024-09-14T07:29:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-09-14T07:29:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_25 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:29:29.251 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_25 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-09-14T07:29:29.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-09-14T07:29:29.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:29.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:29.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:29.396 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_25 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:29.397 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_25 2024-09-14T07:29:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_25 2024-09-14T07:29:29.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-09-14T07:29:29.399 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.445 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-09-14T07:29:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:29.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_26 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_26 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:29:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-09-14T07:29:29.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.489 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-09-14T07:29:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2024-09-14T07:29:29.595 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_26 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_26 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.596 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_26 2024-09-14T07:29:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-09-14T07:29:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_27 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2024-09-14T07:29:29.646 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_27 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-09-14T07:29:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-09-14T07:29:29.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:29.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:29.713 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_27 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_27 2024-09-14T07:29:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_27 2024-09-14T07:29:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-09-14T07:29:29.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.863 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_28 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_28 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:29:29.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-09-14T07:29:29.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:29.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-09-14T07:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_28 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.014 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_28 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_28 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-09-14T07:29:30.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-09-14T07:29:30.067 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_29 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_29 2024-09-14T07:29:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:29:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-09-14T07:29:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-09-14T07:29:30.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:30.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:30.111 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:30.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2024-09-14T07:29:30.212 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_29 2024-09-14T07:29:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:30.213 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_29 2024-09-14T07:29:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_29 2024-09-14T07:29:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-09-14T07:29:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_30 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_30 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-09-14T07:29:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-09-14T07:29:30.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:30.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:30.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:30.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_30 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:30.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_30 2024-09-14T07:29:30.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_30 2024-09-14T07:29:30.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-09-14T07:29:30.410 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_31 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_31 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-09-14T07:29:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-09-14T07:29:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:30.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_31 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:30.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_31 2024-09-14T07:29:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_31 2024-09-14T07:29:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-09-14T07:29:30.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.657 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_32 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_32 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_32 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-09-14T07:29:30.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-09-14T07:29:30.707 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:30.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:30.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_32 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:30.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:30.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_32 2024-09-14T07:29:30.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_32 2024-09-14T07:29:30.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-09-14T07:29:30.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.860 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_33 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:30.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_33 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_33 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-09-14T07:29:30.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:30.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-09-14T07:29:30.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:30.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:30.908 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_33 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:31.010 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_33 2024-09-14T07:29:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_33 2024-09-14T07:29:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-09-14T07:29:31.011 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-09-14T07:29:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_34 2024-09-14T07:29:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2024-09-14T07:29:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_34 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-09-14T07:29:31.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-09-14T07:29:31.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:31.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:31.103 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_34 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_34 2024-09-14T07:29:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_34 2024-09-14T07:29:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-09-14T07:29:31.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_35 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2024-09-14T07:29:31.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_35 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-09-14T07:29:31.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-09-14T07:29:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:31.406 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_35 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:31.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_35 2024-09-14T07:29:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_35 2024-09-14T07:29:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-09-14T07:29:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-09-14T07:29:31.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:31.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_36 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_36 2024-09-14T07:29:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:29:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-09-14T07:29:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.501 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-09-14T07:29:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:31.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_36 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_36 2024-09-14T07:29:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.608 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_36 2024-09-14T07:29:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-09-14T07:29:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_37 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:31.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_37 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-09-14T07:29:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-09-14T07:29:31.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:31.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:31.706 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2024-09-14T07:29:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_37 2024-09-14T07:29:31.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:31.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_37 2024-09-14T07:29:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_37 2024-09-14T07:29:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-09-14T07:29:31.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-09-14T07:29:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_38 2024-09-14T07:29:31.865 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2024-09-14T07:29:31.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:31.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:31.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:29:31.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:31.866 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_38 2024-09-14T07:29:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:31.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:29:31.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-09-14T07:29:31.868 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:31.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-09-14T07:29:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:32.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2024-09-14T07:29:32.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.007 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_38 2024-09-14T07:29:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_38 2024-09-14T07:29:32.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_38 2024-09-14T07:29:32.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-09-14T07:29:32.009 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_39 2024-09-14T07:29:32.060 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_39 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-09-14T07:29:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-09-14T07:29:32.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:32.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:32.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_39 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_39 2024-09-14T07:29:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_39 2024-09-14T07:29:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-09-14T07:29:32.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_40 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.260 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_40 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-09-14T07:29:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-09-14T07:29:32.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:32.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:32.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:32.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_40 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:32.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:32.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_40 2024-09-14T07:29:32.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_40 2024-09-14T07:29:32.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-09-14T07:29:32.409 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-09-14T07:29:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_41 2024-09-14T07:29:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2024-09-14T07:29:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_41 2024-09-14T07:29:32.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:29:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-09-14T07:29:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.500 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-09-14T07:29:32.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:32.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:32.504 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2024-09-14T07:29:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_41 2024-09-14T07:29:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:32.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:32.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:32.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_41 2024-09-14T07:29:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_41 2024-09-14T07:29:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-09-14T07:29:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.655 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-09-14T07:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_42 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_42 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.661 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:29:32.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-09-14T07:29:32.662 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.699 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-09-14T07:29:32.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:32.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:32.704 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2024-09-14T07:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_42 2024-09-14T07:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_42 2024-09-14T07:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_42 2024-09-14T07:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-09-14T07:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-09-14T07:29:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_43 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_43 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:29:32.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-09-14T07:29:32.859 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-09-14T07:29:32.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:32.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:32.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:33.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2024-09-14T07:29:33.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_43 2024-09-14T07:29:33.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:33.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:33.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_43 2024-09-14T07:29:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_43 2024-09-14T07:29:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-09-14T07:29:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-09-14T07:29:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_44 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.057 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_44 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_44 2024-09-14T07:29:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-09-14T07:29:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-09-14T07:29:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:33.099 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:33.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2024-09-14T07:29:33.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_44 2024-09-14T07:29:33.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:33.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:33.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:33.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_44 2024-09-14T07:29:33.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_44 2024-09-14T07:29:33.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-09-14T07:29:33.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-09-14T07:29:33.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:33.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:33.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:33.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:33.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_45 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_45 2024-09-14T07:29:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_45 2024-09-14T07:29:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-09-14T07:29:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-09-14T07:29:33.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:33.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:33.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:33.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2024-09-14T07:29:33.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_45 2024-09-14T07:29:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:33.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_45 2024-09-14T07:29:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_45 2024-09-14T07:29:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-09-14T07:29:33.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.452 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-09-14T07:29:33.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:33.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_46 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:33.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_46 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-09-14T07:29:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.498 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-09-14T07:29:33.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:33.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:33.502 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:33.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2024-09-14T07:29:33.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_46 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_46 2024-09-14T07:29:33.605 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_46 2024-09-14T07:29:33.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-09-14T07:29:33.606 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-09-14T07:29:33.658 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_47 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.659 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_47 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-09-14T07:29:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.698 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-09-14T07:29:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:33.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2024-09-14T07:29:33.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.804 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_47 2024-09-14T07:29:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:33.805 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_47 2024-09-14T07:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_47 2024-09-14T07:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-09-14T07:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-09-14T07:29:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_48 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_48 2024-09-14T07:29:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:29:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-09-14T07:29:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:33.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-09-14T07:29:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_48 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:34.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:34.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_48 2024-09-14T07:29:34.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:34.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:34.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_48 2024-09-14T07:29:34.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-09-14T07:29:34.004 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:34.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_49 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_49 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-09-14T07:29:34.059 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:34.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-09-14T07:29:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:34.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2024-09-14T07:29:34.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_49 2024-09-14T07:29:34.201 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:34.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:34.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:34.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:34.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_49 2024-09-14T07:29:34.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:34.202 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:34.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_49 2024-09-14T07:29:34.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:34.203 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-09-14T07:29:34.250 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-09-14T07:29:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_50 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_50 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_50 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-09-14T07:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:34.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-09-14T07:29:34.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:29:34.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:34.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_50 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_50 2024-09-14T07:29:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:34.402 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_50 2024-09-14T07:29:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-09-14T07:29:34.403 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:34.450 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: delete images during bootstrap' 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T07:29:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T07:29:34.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 07:29:34 TEST: delete images during bootstrap 2024-09-14T07:29:34.456 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 07:29:34 TEST: delete images during bootstrap 2024-09-14T07:29:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_pool_mirror_mode cluster1 mirror image 2024-09-14T07:29:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=image 2024-09-14T07:29:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2024-09-14T07:29:34.496 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:note: changing mirroring mode from pool to image 2024-09-14T07:29:34.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_pool_mirror_mode cluster2 mirror image 2024-09-14T07:29:34.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:34.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.511 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=image 2024-09-14T07:29:34.512 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2024-09-14T07:29:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:note: changing mirroring mode from pool to image 2024-09-14T07:29:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ start_mirror cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set_cluster_instance cluster1 cluster instance 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local val=cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster_var_name=cluster 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local instance_var_name=instance 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cluster=cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' cluster1 = cluster1 ']' 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ instance=0 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval cluster=cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ cluster=cluster1 2024-09-14T07:29:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ eval instance=0 2024-09-14T07:29:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ instance=0 2024-09-14T07:29:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:29:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:29:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=test 2024-09-14T07:29:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 1 10 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_1 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_1 512M 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:29:34.558 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:29:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:29:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:29:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:29:34.559 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_1 2024-09-14T07:29:34.617 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_1 2024-09-14T07:29:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:29:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 journal 2024-09-14T07:29:34.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:29:34.674 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_1': 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 3302faa55e49 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.3302faa55e49 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:29:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:29:34 2024 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:29:34 2024 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:29:34 2024 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 3302faa55e49 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 91092a27-e65a-4d01-a944-5c4a6f69c321 2024-09-14T07:29:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:29:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_1 2024-09-14T07:29:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:34.719 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:29:34.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:29:34.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:29:34.721 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:29:35.020 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6437094790114035) > weight (0.25) 2024-09-14T07:29:35.020 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T07:29:35.021 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T07:29:35.021 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-14T07:29:35.022 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:29:35.021+0000 7f9e3d3d9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 117057) UID: 1000 2024-09-14T07:29:35.038 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 891904 bytes 2024-09-14T07:29:39.228 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 62582784 bytes 2024-09-14T07:29:39.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 64957440 bytes 2024-09-14T07:29:40.134 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-14T07:29:40.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-14T07:29:40.134 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T07:29:40.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:29:40.132+0000 7f39fe030640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 119404) UID: 1000 2024-09-14T07:29:42.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 98617344 bytes 2024-09-14T07:29:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:29:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:29:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:29:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:29:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:29:42.732 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=11822d5ebad8 2024-09-14T07:29:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:29:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:29:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_1 present 11822d5ebad8 2024-09-14T07:29:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11822d5ebad8 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:29:42.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:29:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:29:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:29:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:29:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:29:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=11822d5ebad8 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 11822d5ebad8 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 11822d5ebad8 -o 11822d5ebad8 = 11822d5ebad8 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:29:42.957 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_1 2024-09-14T07:29:42.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:29:42.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:29:42.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:29:42.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2024-09-14T07:29:44.022 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T07:29:44.022 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:29:44.022 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T07:29:44.022 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T07:29:44.023 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T07:29:44.025 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T07:29:44.026 INFO:tasks.rbd_mirror_thrash:waiting for 83 secs before thrashing 2024-09-14T07:29:47.127 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:29:47.124+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b3a8e3c0 finish: resending after timeout 2024-09-14T07:29:47.135 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-14T07:29:47.135 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:29:47.135 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-14T07:29:47.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-14T07:29:47.135 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-14T07:29:47.138 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-14T07:29:47.138 INFO:tasks.rbd_mirror_thrash:waiting for 85 secs before thrashing 2024-09-14T07:29:52.129 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:29:52.128+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b3a8e3c0 finish: resending after timeout 2024-09-14T07:29:57.130 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:29:57.132+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b3a8e3c0 finish: resending after timeout 2024-09-14T07:30:02.132 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:30:02.132+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b3a8e3c0 finish: resending after timeout 2024-09-14T07:30:07.133 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:30:07.132+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b3a8e3c0 finish: resending after timeout 2024-09-14T07:30:10.003 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:30:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:30:10.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:30:10.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:10.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:10.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:30:10.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:30:10.020 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:10.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=11822d5ebad8 2024-09-14T07:30:10.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:30:10.081 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_1 deleted 11822d5ebad8 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11822d5ebad8 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:30:10.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:30:10.184 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=11822d5ebad8 2024-09-14T07:30:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 11822d5ebad8 2024-09-14T07:30:10.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 11822d5ebad8 -o 11822d5ebad8 = 11822d5ebad8 2024-09-14T07:30:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:30:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = present 2024-09-14T07:30:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:30:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:30:11.253 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_1 deleted 11822d5ebad8 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11822d5ebad8 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_1 2024-09-14T07:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_1 2024-09-14T07:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-14T07:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-14T07:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_1 2024-09-14T07:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:30:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2024-09-14T07:30:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-14T07:30:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_1 2024-09-14T07:30:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:30:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:30:11.958 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:30:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_1 2024-09-14T07:30:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_1 2024-09-14T07:30:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-09-14T07:30:12.032 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-09-14T07:30:12.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_2 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_2 512M 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:30:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:30:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:30:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:12.297 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_2 2024-09-14T07:30:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_2 2024-09-14T07:30:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:30:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 journal 2024-09-14T07:30:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:30:12.414 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-09-14T07:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_2': 2024-09-14T07:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 33249fd86b4a 2024-09-14T07:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.33249fd86b4a 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:30:12 2024 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:30:12 2024 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:30:12 2024 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 33249fd86b4a 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 493227db-a32a-4cc5-83cf-132b21cbc1c8 2024-09-14T07:30:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:30:12.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_2 2024-09-14T07:30:12.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:12.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:12.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:12.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:30:12.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:30:12.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:30:12.460 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:30:14.616 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 26464256 bytes 2024-09-14T07:30:15.576 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 32312320 bytes 2024-09-14T07:30:15.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 37942272 bytes 2024-09-14T07:30:20.114 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 104606720 bytes 2024-09-14T07:30:22.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:500 IOs, 150111232 bytes 2024-09-14T07:30:24.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:30:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:30:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T07:30:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:30:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T07:30:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:24.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=118216ceea31 2024-09-14T07:30:24.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:30:24.532 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:30:24.634 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_2 present 118216ceea31 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=118216ceea31 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:30:24.635 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T07:30:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:30:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T07:30:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=118216ceea31 2024-09-14T07:30:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 118216ceea31 2024-09-14T07:30:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 118216ceea31 -o 118216ceea31 = 118216ceea31 2024-09-14T07:30:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:30:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:30:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:30:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_2 2024-09-14T07:30:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:24.702 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2024-09-14T07:30:55.048 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:30:55.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-09-14T07:30:55.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:55.061 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T07:30:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:55.100 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T07:30:55.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:30:55.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:30:55.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:30:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-09-14T07:30:55.205 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:30:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:30:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:30:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:30:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:30:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_2 2024-09-14T07:30:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:30:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:30:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_2 2024-09-14T07:30:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-14T07:30:55.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:30:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_2 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2024-09-14T07:30:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_2 2024-09-14T07:30:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:30:55.306 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:30:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_2 2024-09-14T07:30:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_2 2024-09-14T07:30:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-14T07:30:55.355 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-14T07:30:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_3 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_3 512M 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:30:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_3 2024-09-14T07:30:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_3 2024-09-14T07:30:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:30:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:30:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 journal 2024-09-14T07:30:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:30:55.727 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_3': 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 3335edf70af7 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.3335edf70af7 2024-09-14T07:30:55.767 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:30:55 2024 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:30:55 2024 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:30:55 2024 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 3335edf70af7 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: cfba7362-c04f-46ad-b2a7-a793dc1f1017 2024-09-14T07:30:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:30:55.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_3 2024-09-14T07:30:55.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:30:55.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:30:55.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:30:55.772 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:30:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T07:30:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:30:55.774 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:30:55.841 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:30:55.838+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616ec30 2024-09-14T07:31:00.271 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:31:00.262+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616ec30 2024-09-14T07:31:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:31:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:31:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:31:00.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:31:00.786 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T07:31:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:31:00.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=11829380619 2024-09-14T07:31:00.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:31:00.870 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_3 present 11829380619 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11829380619 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:31:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:31:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T07:31:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:31:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:31:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:31:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T07:31:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=11829380619 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 11829380619 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 11829380619 -o 11829380619 = 11829380619 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_3 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:01.042 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2024-09-14T07:31:02.850 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:02.850+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:07.026 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:31:07.027 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:31:07.027 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-14T07:31:07.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:31:07.026+0000 7f1da34ab640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 111000) UID: 1000 2024-09-14T07:31:07.851 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:07.850+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:09.028 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:31:09.028 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:31:09.028 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:31:09.028 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:31:09.029 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:31:09.032 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:31:09.032 INFO:tasks.rbd_mirror_thrash:waiting for 106 secs before thrashing 2024-09-14T07:31:12.139 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.40256950919896317) > weight (0.25) 2024-09-14T07:31:12.139 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9791123046481517) > weight (0.25) 2024-09-14T07:31:12.139 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5707470482675406) > weight (0.25) 2024-09-14T07:31:12.139 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-09-14T07:31:12.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-09-14T07:31:12.140 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-14T07:31:12.141 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:31:12.138+0000 7fe05822d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 53423) UID: 1000 2024-09-14T07:31:12.853 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:12.850+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:13.247 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:31:13.246+0000 7f01cfe96640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5577db507040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-14T07:31:13.248 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:31:13.246+0000 7f01cfe96640 -1 rbd::mirror::ImageReplayer: 0x5577dbab2000 [3/cfba7362-c04f-46ad-b2a7-a793dc1f1017] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-14T07:31:16.141 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-09-14T07:31:16.141 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:31:16.142 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-09-14T07:31:16.142 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-14T07:31:16.142 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-14T07:31:16.145 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-14T07:31:16.145 INFO:tasks.rbd_mirror_thrash:waiting for 91 secs before thrashing 2024-09-14T07:31:17.854 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:17.850+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:22.855 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:22.854+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:27.857 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:27.854+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:32.858 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:32.853+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:33.781 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:31:33.777+0000 7f7c359f7640 -1 reset not still connected to 0x55ba92a852b0 2024-09-14T07:31:33.782 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:31:33.777+0000 7f7c359f7640 -1 reset not still connected to 0x55ba945b60d0 2024-09-14T07:31:33.782 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:31:33.777+0000 7f7c359f7640 -1 reset not still connected to 0x55ba9842fa00 2024-09-14T07:31:33.782 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:31:33.777+0000 7f7c359f7640 -1 reset not still connected to 0x55ba9e548d00 2024-09-14T07:31:33.782 INFO:tasks.ceph.cluster2.osd.2.smithi169.stderr:2024-09-14T07:31:33.777+0000 7f7c359f7640 -1 reset not still connected to 0x55bab616e4e0 2024-09-14T07:31:37.859 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:37.857+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b4aade00 finish: resending after timeout 2024-09-14T07:31:37.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:37.957+0000 7f6a9640e640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x55a2bdd586c0 handle_update_sync_points: failed to update remote journal client meta for image cfba7362-c04f-46ad-b2a7-a793dc1f1017: (2) No such file or directory 2024-09-14T07:31:37.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:37.957+0000 7f6a9640e640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x55a2b49ddec0 handle_update_sync_points: failed to update client data: (2) No such file or directory 2024-09-14T07:31:37.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:37.957+0000 7f6a9640e640 -1 rbd::mirror::ImageSync: 0x55a2b4b914a0 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2024-09-14T07:31:37.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:37.957+0000 7f6a9640e640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55a2b2bcf1e0 handle_image_sync: failed to sync remote image: (2) No such file or directory 2024-09-14T07:31:38.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:31:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-09-14T07:31:38.026 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T07:31:38.027 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:31:38.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:31:38.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:31:38.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T07:31:38.028 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:31:38.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=11829380619 2024-09-14T07:31:38.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:31:38.104 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_3 deleted 11829380619 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11829380619 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_3 2024-09-14T07:31:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:31:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:31:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_3 2024-09-14T07:31:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-14T07:31:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:31:38.254 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-14T07:31:38.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:31:38.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:31:38.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:31:38.258 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:31:38.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_3 2024-09-14T07:31:38.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:38.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:38.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2024-09-14T07:31:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_3 2024-09-14T07:31:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:31:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:31:38.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_3 2024-09-14T07:31:38.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:38.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_3 2024-09-14T07:31:38.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-14T07:31:38.363 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-14T07:31:38.639 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_4 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_4 512M 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.644 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:31:38.645 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_4 2024-09-14T07:31:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_4 2024-09-14T07:31:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:31:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:31:38.708 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 journal 2024-09-14T07:31:38.761 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:31:38.768 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_4': 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 3359d65eee46 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.3359d65eee46 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:31:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:31:38 2024 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:31:38 2024 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:31:38 2024 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 3359d65eee46 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 27ef8a27-fd4b-4ae0-91dc-f501d7b1e4d8 2024-09-14T07:31:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:31:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_4 2024-09-14T07:31:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:31:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:31:38.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T07:31:38.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:31:38.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:31:41.134 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 23285760 bytes 2024-09-14T07:31:42.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 46286848 bytes 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:31:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:31:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:31:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:31:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:31:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T07:31:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:31:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:31:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:31:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T07:31:43.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:31:43.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=13dc8afae104 2024-09-14T07:31:43.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:31:43.898 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:31:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_4 present 13dc8afae104 2024-09-14T07:31:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:31:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:31:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:31:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=13dc8afae104 2024-09-14T07:31:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:31:44.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:31:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T07:31:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:31:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:31:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:31:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T07:31:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:31:44.068 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=13dc8afae104 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 13dc8afae104 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 13dc8afae104 -o 13dc8afae104 = 13dc8afae104 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_4 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:31:44.069 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2024-09-14T07:31:45.334 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:45.329+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:31:50.336 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:50.333+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:31:55.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:31:55.333+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:32:00.338 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:32:00.333+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:32:05.340 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:32:05.337+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:32:10.341 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:32:10.337+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:32:15.342 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:32:15.336+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2b6bf0300 finish: resending after timeout 2024-09-14T07:32:19.720 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:32:19.728 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T07:32:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:32:19.730 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:32:19.724+0000 7f6111432640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x557091493c20 handle_update_sync_points: failed to update remote journal client meta for image 27ef8a27-fd4b-4ae0-91dc-f501d7b1e4d8: (2) No such file or directory 2024-09-14T07:32:19.730 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:32:19.724+0000 7f6111432640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x557091ad0240 handle_update_sync_points: failed to update client data: (2) No such file or directory 2024-09-14T07:32:19.730 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:32:19.724+0000 7f6111432640 -1 rbd::mirror::ImageSync: 0x557091780b00 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2024-09-14T07:32:19.730 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:32:19.724+0000 7f6111432640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55709125a820 handle_image_sync: failed to sync remote image: (2) No such file or directory 2024-09-14T07:32:19.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=13dc8afae104 2024-09-14T07:32:19.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:32:19.793 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:32:19.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_4 deleted 13dc8afae104 2024-09-14T07:32:19.894 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=13dc8afae104 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:32:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_4 2024-09-14T07:32:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:32:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:32:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_4 2024-09-14T07:32:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-14T07:32:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:32:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_4 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:19.950 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:32:19.951 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2024-09-14T07:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_4 2024-09-14T07:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:32:20.000 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:32:20.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:32:20.001 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_4 2024-09-14T07:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_4 2024-09-14T07:32:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-14T07:32:20.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-14T07:32:20.304 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_5 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_5 512M 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:32:20.308 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:32:20.309 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_5 2024-09-14T07:32:20.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_5 2024-09-14T07:32:20.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:20.369 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:32:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:32:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 journal 2024-09-14T07:32:20.419 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:32:20.426 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-09-14T07:32:20.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_5': 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 336a9d5081bb 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.336a9d5081bb 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:32:20 2024 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:32:20 2024 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:32:20 2024 2024-09-14T07:32:20.466 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 336a9d5081bb 2024-09-14T07:32:20.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:32:20.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:32:20.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 3c8ed8e9-a49c-4d93-ba45-fba642309f13 2024-09-14T07:32:20.467 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:32:20.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_5 2024-09-14T07:32:20.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:20.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:20.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:32:20.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:32:20.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=12 2024-09-14T07:32:20.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:32:20.471 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:32:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 46533632 bytes 2024-09-14T07:32:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 92095488 bytes 2024-09-14T07:32:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 147099648 bytes 2024-09-14T07:32:31.971 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 185665536 bytes 2024-09-14T07:32:32.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:32:32.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:32:32.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:32:32.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:32:32.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T07:32:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:32:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=11825ec63573 2024-09-14T07:32:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:32:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_5 present 11825ec63573 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11825ec63573 2024-09-14T07:32:32.642 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T07:32:32.643 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=11825ec63573 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 11825ec63573 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 11825ec63573 -o 11825ec63573 = 11825ec63573 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:32:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_5 2024-09-14T07:32:32.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:32:32.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:32:32.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:32:32.710 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2024-09-14T07:32:47.147 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6811420396386253) > weight (0.25) 2024-09-14T07:32:47.147 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.32447159819043825) > weight (0.25) 2024-09-14T07:32:47.147 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.41692791683484487) > weight (0.25) 2024-09-14T07:32:47.147 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6016423780551978) > weight (0.25) 2024-09-14T07:32:47.148 INFO:tasks.rbd_mirror_thrash:waiting for 73 secs before thrashing 2024-09-14T07:32:55.033 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9059022742355187) > weight (0.25) 2024-09-14T07:32:55.033 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-14T07:32:55.034 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-14T07:32:55.034 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-14T07:32:55.035 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:32:55.035+0000 7f1d2869f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 130049) UID: 1000 2024-09-14T07:33:01.085 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:33:01.097 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:33:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T07:33:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:33:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T07:33:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T07:33:01.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:33:01.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:33:01.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:33:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_5 2024-09-14T07:33:01.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:01.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:01.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_5 2024-09-14T07:33:01.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-14T07:33:01.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:01.286 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-14T07:33:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:33:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:33:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:33:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_5 2024-09-14T07:33:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:01.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:33:01.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2024-09-14T07:33:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_5 2024-09-14T07:33:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:33:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:33:01.341 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:33:01.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_5 2024-09-14T07:33:01.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:01.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_5 2024-09-14T07:33:01.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-14T07:33:01.391 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-14T07:33:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:33:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:33:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_6 2024-09-14T07:33:01.671 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_6 512M 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:01.672 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_6 2024-09-14T07:33:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_6 2024-09-14T07:33:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:33:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 journal 2024-09-14T07:33:01.783 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:33:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-09-14T07:33:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_6': 2024-09-14T07:33:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 33826af4b3b2 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.33826af4b3b2 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:33:01 2024 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:33:01 2024 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:33:01 2024 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 33826af4b3b2 2024-09-14T07:33:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:33:01.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:33:01.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: a69fd97c-f996-42c4-ae71-87520267edff 2024-09-14T07:33:01.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:33:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_6 2024-09-14T07:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:33:01.835 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=5 2024-09-14T07:33:01.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:33:01.836 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:33:02.034 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-14T07:33:02.035 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:33:02.035 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-14T07:33:02.035 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-14T07:33:02.035 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-14T07:33:02.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-14T07:33:02.038 INFO:tasks.rbd_mirror_thrash:waiting for 105 secs before thrashing 2024-09-14T07:33:06.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:33:06.848 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:06.849 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:33:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:33:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:07.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=1182fe9cc0f7 2024-09-14T07:33:07.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:33:07.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:33:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_6 present 1182fe9cc0f7 2024-09-14T07:33:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:33:07.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=1182fe9cc0f7 2024-09-14T07:33:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:33:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:33:07.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:33:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:33:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:33:07.220 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=1182fe9cc0f7 2024-09-14T07:33:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1182fe9cc0f7 2024-09-14T07:33:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 1182fe9cc0f7 -o 1182fe9cc0f7 = 1182fe9cc0f7 2024-09-14T07:33:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:33:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:33:07.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:07.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_6 2024-09-14T07:33:07.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:07.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:07.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:07.294 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2024-09-14T07:33:10.110 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:33:10.107+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2bfbc6300 finish: resending after timeout 2024-09-14T07:33:15.111 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:33:15.111+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2bfbc6300 finish: resending after timeout 2024-09-14T07:33:20.113 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:33:20.111+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2bfbc6300 finish: resending after timeout 2024-09-14T07:33:25.114 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:33:25.115+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2bfbc6300 finish: resending after timeout 2024-09-14T07:33:30.115 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:33:30.115+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2bfbc6300 finish: resending after timeout 2024-09-14T07:33:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:33:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-09-14T07:33:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:33:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:33:35.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:35.116 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:33:35.115+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x55a2bfbc6300 finish: resending after timeout 2024-09-14T07:33:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=1182fe9cc0f7 2024-09-14T07:33:35.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:33:35.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_6 deleted 1182fe9cc0f7 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=1182fe9cc0f7 2024-09-14T07:33:35.241 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:33:35.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:33:35.242 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:33:35.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:35.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:35.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:33:35.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:33:35.243 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:35.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=1182fe9cc0f7 2024-09-14T07:33:35.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1182fe9cc0f7 2024-09-14T07:33:35.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 1182fe9cc0f7 -o 1182fe9cc0f7 = 1182fe9cc0f7 2024-09-14T07:33:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:33:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = present 2024-09-14T07:33:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:33:35.315 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 1 2024-09-14T07:33:36.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_6 deleted 1182fe9cc0f7 2024-09-14T07:33:36.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:36.316 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:33:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=1182fe9cc0f7 2024-09-14T07:33:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:33:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:33:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_6 2024-09-14T07:33:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_6 2024-09-14T07:33:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-14T07:33:36.320 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:36.566 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_6 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2024-09-14T07:33:36.999 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-09-14T07:33:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_6 2024-09-14T07:33:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:33:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:33:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_6 2024-09-14T07:33:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_6 2024-09-14T07:33:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-14T07:33:37.086 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-14T07:33:37.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_7 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_7 512M 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:37.348 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:33:37.349 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_7 2024-09-14T07:33:37.407 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_7 2024-09-14T07:33:37.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:37.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:37.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:33:37.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:33:37.408 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 journal 2024-09-14T07:33:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:33:37.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_7': 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 33a0aa7cafb6 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.33a0aa7cafb6 2024-09-14T07:33:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:33:37 2024 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:33:37 2024 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:33:37 2024 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 33a0aa7cafb6 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 62c0f27d-ac12-4bbd-82dd-7071c5378559 2024-09-14T07:33:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:33:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_7 2024-09-14T07:33:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:33:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:33:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=7 2024-09-14T07:33:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:33:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:33:41.950 INFO:tasks.ceph.cluster2.osd.1.smithi169.stderr:2024-09-14T07:33:41.946+0000 7f7deb8ac640 -1 reset not still connected to 0x55ca84274ea0 2024-09-14T07:33:42.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 108085248 bytes 2024-09-14T07:33:43.299 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 119666688 bytes 2024-09-14T07:33:44.470 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 137340928 bytes 2024-09-14T07:33:44.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:33:44.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:33:44.523 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:33:44.524 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T07:33:44.525 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:44.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=1182280db334 2024-09-14T07:33:44.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:33:44.585 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_7 present 1182280db334 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=1182280db334 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:33:44.686 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:33:44.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T07:33:44.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:33:44.687 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:33:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:33:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T07:33:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:33:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=1182280db334 2024-09-14T07:33:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1182280db334 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 1182280db334 -o 1182280db334 = 1182280db334 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_7 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:33:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2024-09-14T07:34:00.149 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.862223285425164) > weight (0.25) 2024-09-14T07:34:00.149 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8894141651691124) > weight (0.25) 2024-09-14T07:34:00.149 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-14T07:34:00.149 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-14T07:34:00.150 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons 2024-09-14T07:34:00.151 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:34:00.150+0000 7fe732340640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 123500) UID: 1000 2024-09-14T07:34:03.150 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-14T07:34:03.151 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:34:03.151 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-14T07:34:03.151 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-14T07:34:03.151 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-14T07:34:03.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-14T07:34:03.154 INFO:tasks.rbd_mirror_thrash:waiting for 113 secs before thrashing 2024-09-14T07:34:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:34:13.310 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:34:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:34:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T07:34:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:34:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:34:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:34:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T07:34:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:34:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T07:34:13.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:34:13.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:34:13.361 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:34:13.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-09-14T07:34:13.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:34:13.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:34:13.462 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:34:13.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:34:13.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:34:13.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:34:13.463 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_7 2024-09-14T07:34:13.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:34:13.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:34:13.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_7 2024-09-14T07:34:13.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-14T07:34:13.464 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:34:13.503 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-14T07:34:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:34:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:34:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:34:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:34:13.508 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_7 2024-09-14T07:34:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:34:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2024-09-14T07:34:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_7 2024-09-14T07:34:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:34:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:34:13.560 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:34:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_7 2024-09-14T07:34:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_7 2024-09-14T07:34:13.561 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-14T07:34:13.609 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-14T07:34:13.869 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_8 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_8 512M 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:13.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:34:13.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:34:13.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:34:13.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:13.875 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_8 2024-09-14T07:34:13.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_8 2024-09-14T07:34:13.935 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:13.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:13.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:13.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:34:13.936 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 journal 2024-09-14T07:34:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:34:13.994 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_8': 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 33c14959572b 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.33c14959572b 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:34:14.035 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:34:13 2024 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:34:13 2024 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:34:13 2024 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 33c14959572b 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 4b6f8a04-5b8c-43b5-aa92-150959aae2ac 2024-09-14T07:34:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:34:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_8 2024-09-14T07:34:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:14.039 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:34:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:34:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:34:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:34:14.730 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 6565888 bytes 2024-09-14T07:34:19.673 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 75121664 bytes 2024-09-14T07:34:22.050 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:34:22.051 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:34:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T07:34:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:34:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:34:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:34:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T07:34:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:34:22.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=1182d76cb84e 2024-09-14T07:34:22.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:34:22.117 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_8 present 1182d76cb84e 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=1182d76cb84e 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:34:22.218 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:34:22.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T07:34:22.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:34:22.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:34:22.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:34:22.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T07:34:22.219 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:34:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=1182d76cb84e 2024-09-14T07:34:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1182d76cb84e 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 1182d76cb84e -o 1182d76cb84e = 1182d76cb84e 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_8 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2024-09-14T07:34:47.038 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.679044751959033) > weight (0.25) 2024-09-14T07:34:47.039 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7796515633065182) > weight (0.25) 2024-09-14T07:34:47.039 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4698653456284516) > weight (0.25) 2024-09-14T07:34:47.039 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.43376759888664684) > weight (0.25) 2024-09-14T07:34:47.039 INFO:tasks.rbd_mirror_thrash:waiting for 80 secs before thrashing 2024-09-14T07:34:51.078 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:34:51.090 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:34:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T07:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T07:34:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:34:51.133 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T07:34:51.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:34:51.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:34:51.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:34:51.239 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:34:51.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_8 2024-09-14T07:34:51.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:34:51.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:34:51.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_8 2024-09-14T07:34:51.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-14T07:34:51.240 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:34:51.287 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-14T07:34:51.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:34:51.291 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_8 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2024-09-14T07:34:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_8 2024-09-14T07:34:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:34:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:34:51.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_8 2024-09-14T07:34:51.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:51.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_8 2024-09-14T07:34:51.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-14T07:34:51.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-14T07:34:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:34:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:34:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:34:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_9 2024-09-14T07:34:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_9 512M 2024-09-14T07:34:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:34:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_9 2024-09-14T07:34:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_9 2024-09-14T07:34:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:34:51.715 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:34:51.716 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 journal 2024-09-14T07:34:51.765 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:34:51.771 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_9': 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 33d94617df92 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.33d94617df92 2024-09-14T07:34:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:34:51 2024 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:34:51 2024 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:34:51 2024 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 33d94617df92 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 650632ee-9b17-492e-94cd-42f0570c7807 2024-09-14T07:34:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:34:51.815 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_9 2024-09-14T07:34:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:34:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:34:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:34:51.816 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:34:51.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=11 2024-09-14T07:34:51.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:34:51.817 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:34:53.451 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 24786944 bytes 2024-09-14T07:34:57.107 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 79931392 bytes 2024-09-14T07:34:57.149 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:34:57.144+0000 7fb3f8d94640 -1 reset not still connected to 0x5614bb48b380 2024-09-14T07:34:59.765 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:34:59.760+0000 7fb3f8d94640 -1 reset not still connected to 0x5614bb48b380 2024-09-14T07:35:00.414 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:35:00.408+0000 7fb3f8d94640 -1 reset not still connected to 0x5614bb48b380 2024-09-14T07:35:00.465 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 121866240 bytes 2024-09-14T07:35:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:35:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:35:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:35:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T07:35:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:35:02.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=1182457961c3 2024-09-14T07:35:02.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:35:02.884 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:35:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_9 present 1182457961c3 2024-09-14T07:35:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:35:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:35:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=1182457961c3 2024-09-14T07:35:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:35:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:35:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T07:35:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:35:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:35:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:35:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T07:35:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:35:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=1182457961c3 2024-09-14T07:35:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1182457961c3 2024-09-14T07:35:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 1182457961c3 -o 1182457961c3 = 1182457961c3 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_9 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:03.054 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2024-09-14T07:35:33.109 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:35:33.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-09-14T07:35:33.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:35:33.119 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T07:35:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:35:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T07:35:33.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:35:33.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:35:33.161 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:35:33.263 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:35:33.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_9 2024-09-14T07:35:33.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:35:33.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:35:33.264 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_9 2024-09-14T07:35:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-14T07:35:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:35:33.311 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_9 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:33.318 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2024-09-14T07:35:33.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_9 2024-09-14T07:35:33.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:33.365 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_9 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.366 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_9 2024-09-14T07:35:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-14T07:35:33.415 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-14T07:35:33.676 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for i in `seq 1 10` 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image=image_10 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ create_image cluster2 mirror image_10 512M 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:35:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local size=128 2024-09-14T07:35:33.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -n 512M ']' 2024-09-14T07:35:33.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ size=512M 2024-09-14T07:35:33.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ shift 2024-09-14T07:35:33.681 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_10 2024-09-14T07:35:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ enable_mirror cluster2 mirror image_10 2024-09-14T07:35:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:35:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local mode=journal 2024-09-14T07:35:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 journal 2024-09-14T07:35:33.790 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring enabled 2024-09-14T07:35:33.797 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-09-14T07:35:33.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:rbd image 'image_10': 2024-09-14T07:35:33.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: size 512 MiB in 128 objects 2024-09-14T07:35:33.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: order 22 (4 MiB objects) 2024-09-14T07:35:33.837 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: snapshot_count: 0 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: id: 33f1ced73563 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: block_name_prefix: rbd_data.33f1ced73563 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: format: 2 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: features: layering, exclusive-lock, journaling 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: op_features: 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: flags: 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: create_timestamp: Sat Sep 14 07:35:33 2024 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: access_timestamp: Sat Sep 14 07:35:33 2024 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: modify_timestamp: Sat Sep 14 07:35:33 2024 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: journal: 33f1ced73563 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring state: enabled 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring mode: journal 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring global id: 1f509319-350f-4680-b957-7d78adac394b 2024-09-14T07:35:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout: mirroring primary: true 2024-09-14T07:35:33.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stress_write_image cluster2 mirror image_10 2024-09-14T07:35:33.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:33.841 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:33.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:35:33.842 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-14T07:35:33.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local duration=8 2024-09-14T07:35:33.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:35:33.843 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-09-14T07:35:34.140 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:100 IOs, 11482112 bytes 2024-09-14T07:35:36.102 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:200 IOs, 26025984 bytes 2024-09-14T07:35:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:300 IOs, 68289536 bytes 2024-09-14T07:35:41.800 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:400 IOs, 121380864 bytes 2024-09-14T07:35:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=124 2024-09-14T07:35:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:35:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 124 -eq 124 ']' 2024-09-14T07:35:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:35:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=present 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:35:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:35:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T07:35:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:35:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:35:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:35:41.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:35:41.857 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T07:35:41.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id=11825c6b2c63 2024-09-14T07:35:41.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:35:41.920 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_10 present 11825c6b2c63 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=present 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id=11825c6b2c63 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:35:42.021 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:35:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T07:35:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:35:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:35:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:35:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T07:35:42.022 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:35:42.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id=11825c6b2c63 2024-09-14T07:35:42.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 11825c6b2c63 2024-09-14T07:35:42.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -z 11825c6b2c63 -o 11825c6b2c63 = 11825c6b2c63 2024-09-14T07:35:42.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_state=present 2024-09-14T07:35:42.095 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test present = present 2024-09-14T07:35:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:35:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ disable_mirror cluster2 mirror image_10 2024-09-14T07:35:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:35:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:35:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:35:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2024-09-14T07:35:42.147 INFO:tasks.ceph.cluster2.osd.0.smithi169.stderr:2024-09-14T07:35:42.143+0000 7fb3e614e640 -1 bdev(0x56148c75ac00 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 4 2024-09-14T07:35:42.223 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:35:42.219+0000 7f6aa3c29640 -1 rbd::mirror::ImageReplayer: 0x55a2b8ba1b80 [3/1f509319-350f-4680-b957-7d78adac394b] start: already running 2024-09-14T07:35:56.155 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.763251230763963) > weight (0.25) 2024-09-14T07:35:56.155 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9928783610666319) > weight (0.25) 2024-09-14T07:35:56.155 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4704926598639261) > weight (0.25) 2024-09-14T07:35:56.155 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5130337683225685) > weight (0.25) 2024-09-14T07:35:56.156 INFO:tasks.rbd_mirror_thrash:waiting for 102 secs before thrashing 2024-09-14T07:36:07.040 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-14T07:36:07.041 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-14T07:36:07.041 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-14T07:36:07.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:07.038+0000 7efee9d19640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 130934) UID: 1000 2024-09-14T07:36:11.314 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:Mirroring disabled 2024-09-14T07:36:11.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-09-14T07:36:11.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:11.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:36:11.329 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local state=deleted 2024-09-14T07:36:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:36:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local s 2024-09-14T07:36:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:36:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T07:36:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:36:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T07:36:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:36:11.375 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T07:36:11.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ image_id= 2024-09-14T07:36:11.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-14T07:36:11.379 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0.1 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local test_state=deleted 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image_id= 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_state=deleted 2024-09-14T07:36:11.481 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local current_image_id 2024-09-14T07:36:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ get_image_id cluster1 mirror image_10 2024-09-14T07:36:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local image=image_10 2024-09-14T07:36:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-14T07:36:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-14T07:36:11.529 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ current_image_id= 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n '' 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test deleted = deleted 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ purge_snapshots cluster2 mirror image_10 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:36:11.535 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2024-09-14T07:36:11.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image_retry cluster2 mirror image_10 2024-09-14T07:36:11.589 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:11.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:36:11.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-14T07:36:11.590 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:11.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ remove_image cluster2 mirror image_10 2024-09-14T07:36:11.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:11.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local image=image_10 2024-09-14T07:36:11.591 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-14T07:36:11.641 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-14T07:36:11.897 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-14T07:36:11.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:11.901 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2024-09-14T07:36:11.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ tee -a /tmp/tmp.gBxNEDbwzY/rbd-mirror.test.log 2024-09-14T07:36:11.902 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ date '+%F %T' 2024-09-14T07:36:11.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo 2024-09-14 07:36:11 TEST: check if removed 'images'\''' OMAP are removed 2024-09-14T07:36:11.903 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:2024-09-14 07:36:11 TEST: check if removed images' OMAP are removed 2024-09-14T07:36:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_in_omap cluster1 mirror 2024-09-14T07:36:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2024-09-14T07:36:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local obj_name=rbd_mirroring 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local filter=status_global 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local filter=status_global 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -c status_global 2024-09-14T07:36:11.907 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2024-09-14T07:36:12.041 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-14T07:36:12.042 INFO:teuthology.orchestra.run:waiting for 600 2024-09-14T07:36:12.042 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-14T07:36:12.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-14T07:36:12.042 DEBUG:teuthology.orchestra.run.smithi169:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-14T07:36:12.045 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-14T07:36:12.045 INFO:tasks.rbd_mirror_thrash:waiting for 108 secs before thrashing 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:12.135 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local obj_name=rbd_mirroring 2024-09-14T07:36:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local filter=image_ 2024-09-14T07:36:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T07:36:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local filter=image_ 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2024-09-14T07:36:12.137 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -c image_ 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local obj_name=rbd_mirror_leader 2024-09-14T07:36:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local filter=image_map 2024-09-14T07:36:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T07:36:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local filter=image_map 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster1 2024-09-14T07:36:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -c image_map 2024-09-14T07:36:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T07:36:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2024-09-14T07:36:12.221 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_image_in_omap cluster2 mirror 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local obj_name=rbd_mirroring 2024-09-14T07:36:12.222 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local filter=status_global 2024-09-14T07:36:12.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T07:36:12.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:12.223 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local filter=status_global 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -c status_global 2024-09-14T07:36:12.225 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local obj_name=rbd_mirroring 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local filter=image_ 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T07:36:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local filter=image_ 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -c image_ 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirroring 2024-09-14T07:36:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local pool=mirror 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local obj_name=rbd_mirror_leader 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local filter=image_map 2024-09-14T07:36:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-14T07:36:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ sleep 0 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local filter=image_map 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local cluster=cluster2 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ grep -c image_map 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local pool=mirror 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ local obj_name=rbd_mirror_leader 2024-09-14T07:36:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2024-09-14T07:36:12.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test 0 = 0 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ error_code=0 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set -e 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 0 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ cleanup 0 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local error_code=0 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ set +e 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -ne 0 ']' 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z '' ']' 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster instance 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T07:36:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-14T07:36:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror' does not exist 2024-09-14T07:36:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T07:36:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-14T07:36:13.359 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror' does not exist 2024-09-14T07:36:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T07:36:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-14T07:36:13.592 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c367d640 -1 rbd::mirror::RemotePoolPoller: 0x5577da449520 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T07:36:13.592 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c2e7c640 -1 rbd::mirror::RemotePoolPoller: 0x5577da449520 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T07:36:13.592 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c2e7c640 -1 rbd::mirror::PoolReplayer: 0x5577da45bb00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T07:36:13.593 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9689640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x5577db508d80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:13.593 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9689640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x5577dbb86900 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:13.593 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9689640 -1 rbd::mirror::PoolWatcher: 0x5577db785680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:13.593 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9e8a640 -1 rbd::mirror::PoolWatcher: 0x5577db7850e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:13.594 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c2e7c640 -1 rbd::mirror::PoolWatcher: 0x5577dbb39e00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:13.594 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c2e7c640 -1 rbd::mirror::PoolWatcher: 0x5577dbcbda40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01d0697640 -1 rbd::mirror::InstanceWatcher: 0x5577db590380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10051 object: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01d0697640 -1 rbd::mirror::InstanceWatcher: 0x5577db590700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10051 object: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9689640 -1 rbd::mirror::InstanceWatcher: 0x5577db590380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9e8a640 -1 rbd::mirror::InstanceWatcher: 0x5577db590700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c367d640 -1 rbd::mirror::MirrorStatusUpdater 0x5577db5f65a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c367d640 -1 rbd::mirror::MirrorStatusUpdater 0x5577db5f6360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01cfe96640 -1 rbd::mirror::NamespaceReplayer: 0x5577db5f4000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01cfe96640 -1 rbd::mirror::NamespaceReplayer: 0x5577db5f41a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9689640 -1 rbd::mirror::MirrorStatusUpdater 0x5577db5f6000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01c9689640 -1 rbd::mirror::MirrorStatusUpdater 0x5577db5f6120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01d0697640 -1 rbd::mirror::NamespaceReplayer: 0x5577db5f4000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:13.595 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.590+0000 7f01d0697640 -1 rbd::mirror::NamespaceReplayer: 0x5577db5f41a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:13.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.934+0000 7f01c9e8a640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x5577dbb86b40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:13.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.934+0000 7f01c9689640 -1 rbd::mirror::PoolWatcher: 0x5577dbcbc1e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:13.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.934+0000 7f01c2e7c640 -1 rbd::mirror::PoolWatcher: 0x5577db784000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:13.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01d0697640 -1 rbd::mirror::LeaderWatcher: 0x5577db50ec00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T07:36:13.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01d0697640 -1 rbd::mirror::LeaderWatcher: 0x5577db50ec00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-14T07:36:13.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01cfe96640 -1 rbd::mirror::LeaderWatcher: 0x5577db50ec00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:13.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01cfe96640 -1 rbd::mirror::InstanceWatcher: 0x5577db590000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10051 object: (2) No such file or directory 2024-09-14T07:36:13.942 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01c9e8a640 -1 rbd::mirror::InstanceWatcher: 0x5577db590000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:13.943 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01c367d640 -1 rbd::mirror::MirrorStatusUpdater 0x5577dbfa67e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:13.943 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01d0697640 -1 rbd::mirror::NamespaceReplayer: 0x5577db318340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:13.943 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01c9689640 -1 rbd::mirror::MirrorStatusUpdater 0x5577dbf9c360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:13.943 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:13.938+0000 7f01cfe96640 -1 rbd::mirror::NamespaceReplayer: 0x5577db318340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:14.822 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror_parent' does not exist 2024-09-14T07:36:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ CEPH_ARGS= 2024-09-14T07:36:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-14T07:36:15.106 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6aa3c29640 -1 rbd::mirror::LeaderWatcher: 0x55a2b2d2cc00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2024-09-14T07:36:15.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a9cc1b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55a2b2dae240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:15.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a9cc1b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55a2b31fb8c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:15.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a9d41c640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55a2b31fb680 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:15.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a9cc1b640 -1 rbd::mirror::PoolWatcher: 0x55a2b3482000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:15.107 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a9cc1b640 -1 rbd::mirror::PoolWatcher: 0x55a2b2b39e00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:15.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a9cc1b640 -1 rbd::mirror::PoolWatcher: 0x55a2b2b383c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:15.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a96c0f640 -1 rbd::mirror::PoolWatcher: 0x55a2b3482780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:15.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a96c0f640 -1 rbd::mirror::PoolWatcher: 0x55a2b34821e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:15.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.102+0000 7f6a96c0f640 -1 rbd::mirror::PoolWatcher: 0x55a2b34825a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:15.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.106+0000 7f6aa3c29640 -1 rbd::mirror::LeaderWatcher: 0x55a2b2d2cc00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T07:36:15.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.106+0000 7f6aa3c29640 -1 rbd::mirror::LeaderWatcher: 0x55a2b2d2cc00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2024-09-14T07:36:15.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6111c33640 -1 rbd::mirror::RemotePoolPoller: 0x55708fb69860 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T07:36:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:pool 'mirror_parent' does not exist 2024-09-14T07:36:15.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6111432640 -1 rbd::mirror::RemotePoolPoller: 0x55708fb69860 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T07:36:15.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6111432640 -1 rbd::mirror::PoolReplayer: 0x55708fb7bb00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T07:36:15.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f611e44c640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5084 object: (2) No such file or directory 2024-09-14T07:36:15.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f611e44c640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5084 object: (2) No such file or directory 2024-09-14T07:36:15.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6118440640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:15.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6117c3f640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:15.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6111c33640 -1 rbd::mirror::MirrorStatusUpdater 0x557090d565a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f6111c33640 -1 rbd::mirror::MirrorStatusUpdater 0x557090d56360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f611e44c640 -1 rbd::mirror::NamespaceReplayer: 0x557090d54000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.314+0000 7f611e44c640 -1 rbd::mirror::NamespaceReplayer: 0x557090d541a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.318+0000 7f6118440640 -1 rbd::mirror::MirrorStatusUpdater 0x557090d56000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.318+0000 7f611e44c640 -1 rbd::mirror::NamespaceReplayer: 0x557090d54000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.318+0000 7f6117c3f640 -1 rbd::mirror::MirrorStatusUpdater 0x557090d56120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.318+0000 7f611ec4d640 -1 rbd::mirror::NamespaceReplayer: 0x557090d541a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-14T07:36:15.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirrors cluster1 2024-09-14T07:36:15.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1 2024-09-14T07:36:15.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.336 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster1:0 '' 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:0 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster1:1 '' 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:1 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.339 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster1:2 '' 2024-09-14T07:36:15.340 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:2 2024-09-14T07:36:15.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.342 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster1:3 '' 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster1:3 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirrors cluster2 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:++ seq 0 3 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster2:0 '' 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:0 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster2:1 '' 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:1 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster2:2 '' 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:2 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.344 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ stop_mirror cluster2:3 '' 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local cluster=cluster2:3 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ local sig= 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test -n 1 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ return 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' -z 1 ']' 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ test '' = /tmp/tmp.gBxNEDbwzY 2024-09-14T07:36:15.345 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ rm -Rf /tmp/tmp.gBxNEDbwzY 2024-09-14T07:36:15.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ '[' 0 -eq 0 ']' 2024-09-14T07:36:15.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ echo OK 2024-09-14T07:36:15.436 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stdout:OK 2024-09-14T07:36:15.437 INFO:tasks.workunit.cluster1.client.mirror.smithi169.stderr:+ exit 0 2024-09-14T07:36:15.437 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T07:36:15.437 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2024-09-14T07:36:15.585 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2024-09-14T07:36:15.585 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2024-09-14T07:36:15.864 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f611e44c640 -1 rbd::mirror::LeaderWatcher: 0x557090c28c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:15.865 DEBUG:teuthology.parallel:result is None 2024-09-14T07:36:15.865 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-14T07:36:15.866 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f611e44c640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5084 object: (2) No such file or directory 2024-09-14T07:36:15.866 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f6118440640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:15.866 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f6111432640 -1 rbd::mirror::MirrorStatusUpdater 0x5570916d27e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.867 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f611e44c640 -1 rbd::mirror::NamespaceReplayer: 0x557090a341a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.868 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f6117c3f640 -1 rbd::mirror::MirrorStatusUpdater 0x5570916ae360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.868 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.862+0000 7f611ec4d640 -1 rbd::mirror::NamespaceReplayer: 0x557090a341a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.872 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f611e44c640 -1 rbd::mirror::LeaderWatcher: 0x557090f1d200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:15.872 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f6104418640 -1 rbd::mirror::RemotePoolPoller: 0x55708fb680d0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2024-09-14T07:36:15.872 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f6104418640 -1 rbd::mirror::RemotePoolPoller: 0x55708fb680d0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2024-09-14T07:36:15.872 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f6104418640 -1 rbd::mirror::PoolReplayer: 0x557090c28f00 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2024-09-14T07:36:15.873 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f611ec4d640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5087 object: (2) No such file or directory 2024-09-14T07:36:15.873 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f610a424640 -1 rbd::mirror::InstanceWatcher: 0x557090ce0a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:15.874 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f610ac25640 -1 rbd::mirror::MirrorStatusUpdater 0x557090efa900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:15.874 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:15.870+0000 7f611ec4d640 -1 rbd::mirror::NamespaceReplayer: 0x557090d544e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:15.876 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-14T07:36:15.876 DEBUG:teuthology.orchestra.run.smithi169:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-14T07:36:15.929 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-14T07:36:15.929 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2024-09-14T07:36:15.940 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2024-09-14T07:36:15.940 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 22 2024-09-14T07:36:15.940 INFO:tasks.rbd_mirror_thrash:done joining 2024-09-14T07:36:15.940 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2024-09-14T07:36:15.966 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2024-09-14T07:36:15.966 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 22 2024-09-14T07:36:15.967 INFO:tasks.rbd_mirror_thrash:done joining 2024-09-14T07:36:15.967 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.000 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2024-09-14T07:36:16.000 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.001 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.998+0000 7faf0eb7d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 131027) UID: 1000 2024-09-14T07:36:16.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.998+0000 7faf0e37c640 -1 rbd::mirror::LeaderWatcher: 0x56385b797200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.998+0000 7faf0db7b640 -1 rbd::mirror::InstanceWatcher: 0x56385b584380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13131 object: (2) No such file or directory 2024-09-14T07:36:16.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:15.998+0000 7faf0db7b640 -1 rbd::mirror::InstanceWatcher: 0x56385b584a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13131 object: (2) No such file or directory 2024-09-14T07:36:16.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0db7b640 -1 rbd::mirror::InstanceWatcher: 0x56385b584700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13136 object: (2) No such file or directory 2024-09-14T07:36:16.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf07b6f640 -1 rbd::mirror::InstanceWatcher: 0x56385b584380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf07b6f640 -1 rbd::mirror::InstanceWatcher: 0x56385b584a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faefa354640 -1 rbd::mirror::InstanceWatcher: 0x56385b584700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf00b61640 -1 rbd::mirror::MirrorStatusUpdater 0x56385b5da5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf00b61640 -1 rbd::mirror::MirrorStatusUpdater 0x56385b5da360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0e37c640 -1 rbd::mirror::NamespaceReplayer: 0x56385b5d8000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0e37c640 -1 rbd::mirror::NamespaceReplayer: 0x56385b5d81a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faef3346640 -1 rbd::mirror::MirrorStatusUpdater 0x56385b7d87e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf07b6f640 -1 rbd::mirror::MirrorStatusUpdater 0x56385b5da000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0db7b640 -1 rbd::mirror::NamespaceReplayer: 0x56385b2d81a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf07b6f640 -1 rbd::mirror::MirrorStatusUpdater 0x56385b5da120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0db7b640 -1 rbd::mirror::NamespaceReplayer: 0x56385b5d8000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0db7b640 -1 rbd::mirror::NamespaceReplayer: 0x56385b5d81a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faef9b53640 -1 rbd::mirror::MirrorStatusUpdater 0x56385b7d0900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.002+0000 7faf0e37c640 -1 rbd::mirror::NamespaceReplayer: 0x56385b2d81a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.010 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.006+0000 7faf0e37c640 -1 rbd::mirror::LeaderWatcher: 0x56385b4e4c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.011 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.006+0000 7faf0e37c640 -1 rbd::mirror::InstanceWatcher: 0x56385b584000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13131 object: (2) No such file or directory 2024-09-14T07:36:16.011 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.010+0000 7faf0736e640 -1 rbd::mirror::InstanceWatcher: 0x56385b584000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.012 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.010+0000 7faf01362640 -1 rbd::mirror::MirrorStatusUpdater 0x56385bf787e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.012 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.010+0000 7faf0e37c640 -1 rbd::mirror::NamespaceReplayer: 0x56385b2d8340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.013 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.010+0000 7faf07b6f640 -1 rbd::mirror::MirrorStatusUpdater 0x56385bf5c480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.013 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.010+0000 7faf0db7b640 -1 rbd::mirror::NamespaceReplayer: 0x56385b2d8340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.034 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2024-09-14T07:36:16.034 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.046 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2024-09-14T07:36:16.046 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.076 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f7283467640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 132680) UID: 1000 2024-09-14T07:36:16.077 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f7282465640 -1 rbd::mirror::LeaderWatcher: 0x55eb62bd3200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.078 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f727c459640 -1 rbd::mirror::InstanceWatcher: 0x55eb629c0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.078 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f727c459640 -1 rbd::mirror::InstanceWatcher: 0x55eb629c0700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.078 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f7282465640 -1 rbd::mirror::InstanceWatcher: 0x55eb62e86000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13237 object: (2) No such file or directory 2024-09-14T07:36:16.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f727544b640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb62a22360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f727544b640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb62a225a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f7282c66640 -1 rbd::mirror::NamespaceReplayer: 0x55eb62a20000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f7282c66640 -1 rbd::mirror::NamespaceReplayer: 0x55eb62a201a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f726ec3e640 -1 rbd::mirror::InstanceWatcher: 0x55eb62e86000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f727bc58640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb62a22000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.074+0000 7f7282c66640 -1 rbd::mirror::NamespaceReplayer: 0x55eb62a20000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.078+0000 7f7268431640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb62c586c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.078+0000 7f7282465640 -1 rbd::mirror::NamespaceReplayer: 0x55eb627301a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.078+0000 7f726e43d640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb62c0ea20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.081 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.078+0000 7f7282c66640 -1 rbd::mirror::NamespaceReplayer: 0x55eb627301a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.085 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.082+0000 7f7282c66640 -1 rbd::mirror::LeaderWatcher: 0x55eb62912c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.086 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.082+0000 7f727bc58640 -1 rbd::mirror::InstanceWatcher: 0x55eb629c0000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.086 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.082+0000 7f7275c4c640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb633ce7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.086 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.082+0000 7f7282465640 -1 rbd::mirror::NamespaceReplayer: 0x55eb62730340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.087 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.082+0000 7f727c459640 -1 rbd::mirror::MirrorStatusUpdater 0x55eb633ac360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.087 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.082+0000 7f7282c66640 -1 rbd::mirror::NamespaceReplayer: 0x55eb62730340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.107 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2024-09-14T07:36:16.107 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.134 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2024-09-14T07:36:16.134 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.152 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.150+0000 7f01d0e98640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 91865) UID: 1000 2024-09-14T07:36:16.153 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.150+0000 7f01cfe96640 -1 rbd::mirror::LeaderWatcher: 0x5577db7af200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.150+0000 7f01cfe96640 -1 rbd::mirror::InstanceWatcher: 0x5577dba7a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10054 object: (2) No such file or directory 2024-09-14T07:36:16.155 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.150+0000 7f01bc66f640 -1 rbd::mirror::InstanceWatcher: 0x5577dba7a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.155 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.154+0000 7f01b5661640 -1 rbd::mirror::MirrorStatusUpdater 0x5577db5f66c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.156 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.154+0000 7f01cfe96640 -1 rbd::mirror::NamespaceReplayer: 0x5577db3181a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.156 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.154+0000 7f01bbe6e640 -1 rbd::mirror::MirrorStatusUpdater 0x5577db7f2480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.156 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.154+0000 7f01d0697640 -1 rbd::mirror::NamespaceReplayer: 0x5577db3181a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.180 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2024-09-14T07:36:16.180 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.191 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2024-09-14T07:36:16.191 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.224 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2849c59640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 130140) UID: 1000 2024-09-14T07:36:16.225 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2835430640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55d2cc516480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:16.225 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2849458640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13080 object: (2) No such file or directory 2024-09-14T07:36:16.225 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2849458640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13080 object: (2) No such file or directory 2024-09-14T07:36:16.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2834c2f640 -1 rbd::mirror::PoolWatcher: 0x55d2cc804d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:16.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f282ec23640 -1 rbd::mirror::PoolWatcher: 0x55d2cc804960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:16.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2842c4b640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f284244a640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2848c57640 -1 rbd::mirror::LeaderWatcher: 0x55d2cc527200 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T07:36:16.227 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2848c57640 -1 rbd::mirror::LeaderWatcher: 0x55d2cc527200 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-14T07:36:16.227 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.222+0000 7f2849458640 -1 rbd::mirror::LeaderWatcher: 0x55d2cc527200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.231 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f283bc3d640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cc37e5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.231 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f283bc3d640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cc37e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.231 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f2849458640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc37c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.231 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f2849458640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc37c1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.232 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f2848c57640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13083 object: (2) No such file or directory 2024-09-14T07:36:16.232 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f2842c4b640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cc37e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.232 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f2849458640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc37c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.232 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f284244a640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cc37e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.232 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.226+0000 7f2849458640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc37c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.232 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.230+0000 7f2835430640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.230+0000 7f282e422640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cc5a86c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.230+0000 7f2848c57640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc7e2000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.230+0000 7f2834c2f640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cc570a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.230+0000 7f2849458640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc7e2000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.236 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f2848c57640 -1 rbd::mirror::LeaderWatcher: 0x55d2cc27cc00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.237 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f2848c57640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.13080 object: (2) No such file or directory 2024-09-14T07:36:16.238 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f2842c4b640 -1 rbd::mirror::InstanceWatcher: 0x55d2cc306000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f283c43e640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2ccd187e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f2848c57640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc0701a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f284244a640 -1 rbd::mirror::MirrorStatusUpdater 0x55d2cccf2360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.234+0000 7f2849458640 -1 rbd::mirror::NamespaceReplayer: 0x55d2cc0701a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.260 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2024-09-14T07:36:16.260 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.273 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2024-09-14T07:36:16.273 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.304 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6aa442a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 67921) UID: 1000 2024-09-14T07:36:16.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6a8fc01640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55a2b2daf200 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-14T07:36:16.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4482 object: (2) No such file or directory 2024-09-14T07:36:16.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4482 object: (2) No such file or directory 2024-09-14T07:36:16.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6a8f400640 -1 rbd::mirror::PoolWatcher: 0x55a2b2b39680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:16.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6a88bf3640 -1 rbd::mirror::PoolWatcher: 0x55a2b2b392c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-14T07:36:16.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6a9d41c640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.302+0000 7f6a9d41c640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3428640 -1 rbd::mirror::LeaderWatcher: 0x55a2b3017200 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3428640 -1 rbd::mirror::LeaderWatcher: 0x55a2b3017200 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a9640e640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b2e465a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a9640e640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b2e46360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3c29640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2e44000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3c29640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2e441a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3c29640 -1 rbd::mirror::LeaderWatcher: 0x55a2b3017200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a9d41c640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b2e46000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a9d41c640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b2e46120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3428640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2e44000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3428640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2e441a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3c29640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4485 object: (2) No such file or directory 2024-09-14T07:36:16.310 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a8fc01640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a88bf3640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b2e466c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3c29640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2e444e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6a8f400640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b2ff2d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.306+0000 7f6aa3428640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2e444e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6aa3428640 -1 rbd::mirror::LeaderWatcher: 0x55a2b2d2cc00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6aa3428640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4482 object: (2) No such file or directory 2024-09-14T07:36:16.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6a9cc1b640 -1 rbd::mirror::InstanceWatcher: 0x55a2b2de2000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6a96c0f640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b37e27e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6aa3428640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2b2e1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6a9d41c640 -1 rbd::mirror::MirrorStatusUpdater 0x55a2b379a480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi169.stderr:2024-09-14T07:36:16.314+0000 7f6aa3c29640 -1 rbd::mirror::NamespaceReplayer: 0x55a2b2b2e1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.346 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2024-09-14T07:36:16.346 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.359 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2024-09-14T07:36:16.360 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.392 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi169.stderr:2024-09-14T07:36:16.390+0000 7f611f44e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 122080) UID: 1000 2024-09-14T07:36:16.416 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2024-09-14T07:36:16.416 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.431 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2024-09-14T07:36:16.431 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.464 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91d0ced640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 132071) UID: 1000 2024-09-14T07:36:16.465 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91cfceb640 -1 rbd::mirror::LeaderWatcher: 0x55d87a8ed200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.466 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91d04ec640 -1 rbd::mirror::InstanceWatcher: 0x55d87a680380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5514 object: (2) No such file or directory 2024-09-14T07:36:16.466 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91d04ec640 -1 rbd::mirror::InstanceWatcher: 0x55d87a680700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5514 object: (2) No such file or directory 2024-09-14T07:36:16.467 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91cfceb640 -1 rbd::mirror::InstanceWatcher: 0x55d87ab8c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5517 object: (2) No such file or directory 2024-09-14T07:36:16.467 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91c9cdf640 -1 rbd::mirror::InstanceWatcher: 0x55d87a680380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.467 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.462+0000 7f91c94de640 -1 rbd::mirror::InstanceWatcher: 0x55d87a680700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.467 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91c2cd1640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87a6f45a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91bbcc3640 -1 rbd::mirror::InstanceWatcher: 0x55d87ab8c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91c2cd1640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87a6f4360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91cfceb640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a6f2000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91cfceb640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a6f21a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91c94de640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87a6f4000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.468 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91c94de640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87a6f4120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91d04ec640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a6f2000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91d04ec640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a6f21a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91b5cb7640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87a9147e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91cfceb640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a6f2680 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91bc4c4640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87a8caa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.469 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.466+0000 7f91d04ec640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a6f2680 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.473 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.470+0000 7f91cfceb640 -1 rbd::mirror::LeaderWatcher: 0x55d87a5e0c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.475 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.470+0000 7f91cfceb640 -1 rbd::mirror::InstanceWatcher: 0x55d87a680000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5514 object: (2) No such file or directory 2024-09-14T07:36:16.476 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.474+0000 7f91c9cdf640 -1 rbd::mirror::InstanceWatcher: 0x55d87a680000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.476 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.474+0000 7f91c34d2640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87b07a7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.477 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.474+0000 7f91d04ec640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a5241a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.477 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.474+0000 7f91c94de640 -1 rbd::mirror::MirrorStatusUpdater 0x55d87b034480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.477 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi169.stderr:2024-09-14T07:36:16.474+0000 7f91cfceb640 -1 rbd::mirror::NamespaceReplayer: 0x55d87a5241a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.503 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2024-09-14T07:36:16.503 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-14T07:36:16.517 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2024-09-14T07:36:16.517 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:16.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.550+0000 7f0a320fd640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 133798) UID: 1000 2024-09-14T07:36:16.553 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.550+0000 7f0a310fb640 -1 rbd::mirror::LeaderWatcher: 0x5641a86b5200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.554 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.550+0000 7f0a310fb640 -1 rbd::mirror::InstanceWatcher: 0x5641a8962000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5611 object: (2) No such file or directory 2024-09-14T07:36:16.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a1d8d4640 -1 rbd::mirror::InstanceWatcher: 0x5641a8962000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.557 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a170c7640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a86ea7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.557 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a318fc640 -1 rbd::mirror::NamespaceReplayer: 0x5641a81ec1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a1d0d3640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a86c2a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a318fc640 -1 rbd::mirror::NamespaceReplayer: 0x5641a81ec1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a310fb640 -1 rbd::mirror::InstanceWatcher: 0x5641a8486380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5600 object: (2) No such file or directory 2024-09-14T07:36:16.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a310fb640 -1 rbd::mirror::InstanceWatcher: 0x5641a8486a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5600 object: (2) No such file or directory 2024-09-14T07:36:16.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a2a8ee640 -1 rbd::mirror::InstanceWatcher: 0x5641a8486380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.554+0000 7f0a2b0ef640 -1 rbd::mirror::InstanceWatcher: 0x5641a8486a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a248e2640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a84fa360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a248e2640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a84fa5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a310fb640 -1 rbd::mirror::NamespaceReplayer: 0x5641a84f81a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a310fb640 -1 rbd::mirror::NamespaceReplayer: 0x5641a84f8000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a2a8ee640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a84fa120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a2a8ee640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a84fa000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a318fc640 -1 rbd::mirror::NamespaceReplayer: 0x5641a84f81a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a318fc640 -1 rbd::mirror::NamespaceReplayer: 0x5641a84f8000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.558+0000 7f0a318fc640 -1 rbd::mirror::LeaderWatcher: 0x5641a83e8c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-14T07:36:16.564 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.562+0000 7f0a310fb640 -1 rbd::mirror::InstanceWatcher: 0x5641a8486000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5600 object: (2) No such file or directory 2024-09-14T07:36:16.564 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.562+0000 7f0a2b0ef640 -1 rbd::mirror::InstanceWatcher: 0x5641a8486000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-14T07:36:16.565 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.562+0000 7f0a248e2640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a8e967e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.565 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.562+0000 7f0a310fb640 -1 rbd::mirror::NamespaceReplayer: 0x5641a81ec340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.566 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.562+0000 7f0a2a8ee640 -1 rbd::mirror::MirrorStatusUpdater 0x5641a8e50480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-14T07:36:16.566 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi169.stderr:2024-09-14T07:36:16.562+0000 7f0a318fc640 -1 rbd::mirror::NamespaceReplayer: 0x5641a81ec340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-14T07:36:16.591 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2024-09-14T07:36:16.591 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-14T07:36:16.604 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-14T07:36:16.604 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T07:36:16.943 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T07:36:16.944 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T07:36:16.962 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":1743,"stamp":"2024-09-14T07:36:15.918860+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":224,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4200280,"kb_used_data":15516,"kb_used_omap":456,"kb_used_meta":4184247,"kb_avail":278915240,"statfs":{"total":289910292480,"available":285609205760,"internally_reserved":0,"allocated":15888384,"data_stored":8067798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":467391,"internal_metadata":4284669505},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":50,"apply_latency_ms":50,"commit_latency_ns":50000000,"apply_latency_ns":50000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504383574,"num_objects":-168,"num_object_clones":-1,"num_object_copies":-336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-168,"num_whiteouts":-1,"num_read":-826418,"num_read_kb":-48226183,"num_write":-647404,"num_write_kb":-22868151,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002093"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.203721+0000","last_change":"2024-09-14T06:52:02.054336+0000","last_active":"2024-09-14T07:36:13.203721+0000","last_peered":"2024-09-14T07:36:13.203721+0000","last_clean":"2024-09-14T07:36:13.203721+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T07:36:13.203721+0000","last_undegraded":"2024-09-14T07:36:13.203721+0000","last_fullsized":"2024-09-14T07:36:13.203721+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:38:29.413964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.204362+0000","last_change":"2024-09-14T06:52:02.054459+0000","last_active":"2024-09-14T07:36:13.204362+0000","last_peered":"2024-09-14T07:36:13.204362+0000","last_clean":"2024-09-14T07:36:13.204362+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T07:36:13.204362+0000","last_undegraded":"2024-09-14T07:36:13.204362+0000","last_fullsized":"2024-09-14T07:36:13.204362+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:46:17.306187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.210522+0000","last_change":"2024-09-14T06:52:02.055138+0000","last_active":"2024-09-14T07:36:13.210522+0000","last_peered":"2024-09-14T07:36:13.210522+0000","last_clean":"2024-09-14T07:36:13.210522+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T07:36:13.210522+0000","last_undegraded":"2024-09-14T07:36:13.210522+0000","last_fullsized":"2024-09-14T07:36:13.210522+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:01:29.208041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.193329+0000","last_change":"2024-09-14T06:52:02.053552+0000","last_active":"2024-09-14T07:36:13.193329+0000","last_peered":"2024-09-14T07:36:13.193329+0000","last_clean":"2024-09-14T07:36:13.193329+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T07:36:13.193329+0000","last_undegraded":"2024-09-14T07:36:13.193329+0000","last_fullsized":"2024-09-14T07:36:13.193329+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:07.068171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":837,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.298173+0000","last_change":"2024-09-14T06:52:02.063567+0000","last_active":"2024-09-14T07:36:13.298173+0000","last_peered":"2024-09-14T07:36:13.298173+0000","last_clean":"2024-09-14T07:36:13.298173+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T07:36:13.298173+0000","last_undegraded":"2024-09-14T07:36:13.298173+0000","last_fullsized":"2024-09-14T07:36:13.298173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:29.285798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.992860+0000","last_change":"2024-09-14T06:52:02.240204+0000","last_active":"2024-09-14T07:36:13.992860+0000","last_peered":"2024-09-14T07:36:13.992860+0000","last_clean":"2024-09-14T07:36:13.992860+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T07:36:13.992860+0000","last_undegraded":"2024-09-14T07:36:13.992860+0000","last_fullsized":"2024-09-14T07:36:13.992860+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:50.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":836,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.977367+0000","last_change":"2024-09-14T06:52:02.240272+0000","last_active":"2024-09-14T07:36:13.977367+0000","last_peered":"2024-09-14T07:36:13.977367+0000","last_clean":"2024-09-14T07:36:13.977367+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T07:36:13.977367+0000","last_undegraded":"2024-09-14T07:36:13.977367+0000","last_fullsized":"2024-09-14T07:36:13.977367+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:51:25.088755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":836,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.203438+0000","last_change":"2024-09-14T06:52:02.053793+0000","last_active":"2024-09-14T07:36:13.203438+0000","last_peered":"2024-09-14T07:36:13.203438+0000","last_clean":"2024-09-14T07:36:13.203438+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T07:36:13.203438+0000","last_undegraded":"2024-09-14T07:36:13.203438+0000","last_fullsized":"2024-09-14T07:36:13.203438+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:01:49.640848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":904,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.197221+0000","last_change":"2024-09-14T06:51:56.116608+0000","last_active":"2024-09-14T07:36:13.197221+0000","last_peered":"2024-09-14T07:36:13.197221+0000","last_clean":"2024-09-14T07:36:13.197221+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T07:36:13.197221+0000","last_undegraded":"2024-09-14T07:36:13.197221+0000","last_fullsized":"2024-09-14T07:36:13.197221+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_clean_scrub_stamp":"2024-09-14T06:51:55.005796+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:26.973731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706201,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1530372,"kb_used_data":4784,"kb_used_omap":123,"kb_used_meta":1525444,"kb_avail":92841468,"statfs":{"total":96636764160,"available":95069663232,"internally_reserved":0,"allocated":4898816,"data_stored":2295681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":126718,"internal_metadata":1562054914},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706200,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1470156,"kb_used_data":5368,"kb_used_omap":128,"kb_used_meta":1464639,"kb_avail":92901684,"statfs":{"total":96636764160,"available":95131324416,"internally_reserved":0,"allocated":5496832,"data_stored":2886068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":131915,"internal_metadata":1499790517},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706200,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1199752,"kb_used_data":5364,"kb_used_omap":203,"kb_used_meta":1194164,"kb_avail":93172088,"statfs":{"total":96636764160,"available":95408218112,"internally_reserved":0,"allocated":5492736,"data_stored":2886049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208758,"internal_metadata":1222824074},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:36:16.963 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T07:36:17.265 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T07:36:17.266 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T07:36:17.284 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":1743,"stamp":"2024-09-14T07:36:15.918860+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":224,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4200280,"kb_used_data":15516,"kb_used_omap":456,"kb_used_meta":4184247,"kb_avail":278915240,"statfs":{"total":289910292480,"available":285609205760,"internally_reserved":0,"allocated":15888384,"data_stored":8067798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":467391,"internal_metadata":4284669505},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":50,"apply_latency_ms":50,"commit_latency_ns":50000000,"apply_latency_ns":50000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504383574,"num_objects":-168,"num_object_clones":-1,"num_object_copies":-336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-168,"num_whiteouts":-1,"num_read":-826418,"num_read_kb":-48226183,"num_write":-647404,"num_write_kb":-22868151,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002093"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.203721+0000","last_change":"2024-09-14T06:52:02.054336+0000","last_active":"2024-09-14T07:36:13.203721+0000","last_peered":"2024-09-14T07:36:13.203721+0000","last_clean":"2024-09-14T07:36:13.203721+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T07:36:13.203721+0000","last_undegraded":"2024-09-14T07:36:13.203721+0000","last_fullsized":"2024-09-14T07:36:13.203721+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:38:29.413964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.204362+0000","last_change":"2024-09-14T06:52:02.054459+0000","last_active":"2024-09-14T07:36:13.204362+0000","last_peered":"2024-09-14T07:36:13.204362+0000","last_clean":"2024-09-14T07:36:13.204362+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T07:36:13.204362+0000","last_undegraded":"2024-09-14T07:36:13.204362+0000","last_fullsized":"2024-09-14T07:36:13.204362+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:46:17.306187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.210522+0000","last_change":"2024-09-14T06:52:02.055138+0000","last_active":"2024-09-14T07:36:13.210522+0000","last_peered":"2024-09-14T07:36:13.210522+0000","last_clean":"2024-09-14T07:36:13.210522+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T07:36:13.210522+0000","last_undegraded":"2024-09-14T07:36:13.210522+0000","last_fullsized":"2024-09-14T07:36:13.210522+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:01:29.208041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.193329+0000","last_change":"2024-09-14T06:52:02.053552+0000","last_active":"2024-09-14T07:36:13.193329+0000","last_peered":"2024-09-14T07:36:13.193329+0000","last_clean":"2024-09-14T07:36:13.193329+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T07:36:13.193329+0000","last_undegraded":"2024-09-14T07:36:13.193329+0000","last_fullsized":"2024-09-14T07:36:13.193329+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:07.068171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":837,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.298173+0000","last_change":"2024-09-14T06:52:02.063567+0000","last_active":"2024-09-14T07:36:13.298173+0000","last_peered":"2024-09-14T07:36:13.298173+0000","last_clean":"2024-09-14T07:36:13.298173+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T07:36:13.298173+0000","last_undegraded":"2024-09-14T07:36:13.298173+0000","last_fullsized":"2024-09-14T07:36:13.298173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:29.285798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.992860+0000","last_change":"2024-09-14T06:52:02.240204+0000","last_active":"2024-09-14T07:36:13.992860+0000","last_peered":"2024-09-14T07:36:13.992860+0000","last_clean":"2024-09-14T07:36:13.992860+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T07:36:13.992860+0000","last_undegraded":"2024-09-14T07:36:13.992860+0000","last_fullsized":"2024-09-14T07:36:13.992860+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:50.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":836,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.977367+0000","last_change":"2024-09-14T06:52:02.240272+0000","last_active":"2024-09-14T07:36:13.977367+0000","last_peered":"2024-09-14T07:36:13.977367+0000","last_clean":"2024-09-14T07:36:13.977367+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T07:36:13.977367+0000","last_undegraded":"2024-09-14T07:36:13.977367+0000","last_fullsized":"2024-09-14T07:36:13.977367+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:51:25.088755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":836,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.203438+0000","last_change":"2024-09-14T06:52:02.053793+0000","last_active":"2024-09-14T07:36:13.203438+0000","last_peered":"2024-09-14T07:36:13.203438+0000","last_clean":"2024-09-14T07:36:13.203438+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T07:36:13.203438+0000","last_undegraded":"2024-09-14T07:36:13.203438+0000","last_fullsized":"2024-09-14T07:36:13.203438+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:01:49.640848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":904,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.197221+0000","last_change":"2024-09-14T06:51:56.116608+0000","last_active":"2024-09-14T07:36:13.197221+0000","last_peered":"2024-09-14T07:36:13.197221+0000","last_clean":"2024-09-14T07:36:13.197221+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T07:36:13.197221+0000","last_undegraded":"2024-09-14T07:36:13.197221+0000","last_fullsized":"2024-09-14T07:36:13.197221+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_clean_scrub_stamp":"2024-09-14T06:51:55.005796+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:26.973731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706201,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1530372,"kb_used_data":4784,"kb_used_omap":123,"kb_used_meta":1525444,"kb_avail":92841468,"statfs":{"total":96636764160,"available":95069663232,"internally_reserved":0,"allocated":4898816,"data_stored":2295681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":126718,"internal_metadata":1562054914},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706200,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1470156,"kb_used_data":5368,"kb_used_omap":128,"kb_used_meta":1464639,"kb_avail":92901684,"statfs":{"total":96636764160,"available":95131324416,"internally_reserved":0,"allocated":5496832,"data_stored":2886068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":131915,"internal_metadata":1499790517},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706200,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1199752,"kb_used_data":5364,"kb_used_omap":203,"kb_used_meta":1194164,"kb_avail":93172088,"statfs":{"total":96636764160,"available":95408218112,"internally_reserved":0,"allocated":5492736,"data_stored":2886049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208758,"internal_metadata":1222824074},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:36:17.285 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-14T07:36:17.286 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T07:36:17.591 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T07:36:17.591 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T07:36:17.610 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":1743,"stamp":"2024-09-14T07:36:15.918860+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":224,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4200280,"kb_used_data":15516,"kb_used_omap":456,"kb_used_meta":4184247,"kb_avail":278915240,"statfs":{"total":289910292480,"available":285609205760,"internally_reserved":0,"allocated":15888384,"data_stored":8067798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":467391,"internal_metadata":4284669505},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":50,"apply_latency_ms":50,"commit_latency_ns":50000000,"apply_latency_ns":50000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504383574,"num_objects":-168,"num_object_clones":-1,"num_object_copies":-336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-168,"num_whiteouts":-1,"num_read":-826418,"num_read_kb":-48226183,"num_write":-647404,"num_write_kb":-22868151,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002093"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.203721+0000","last_change":"2024-09-14T06:52:02.054336+0000","last_active":"2024-09-14T07:36:13.203721+0000","last_peered":"2024-09-14T07:36:13.203721+0000","last_clean":"2024-09-14T07:36:13.203721+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T07:36:13.203721+0000","last_undegraded":"2024-09-14T07:36:13.203721+0000","last_fullsized":"2024-09-14T07:36:13.203721+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:38:29.413964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.204362+0000","last_change":"2024-09-14T06:52:02.054459+0000","last_active":"2024-09-14T07:36:13.204362+0000","last_peered":"2024-09-14T07:36:13.204362+0000","last_clean":"2024-09-14T07:36:13.204362+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T07:36:13.204362+0000","last_undegraded":"2024-09-14T07:36:13.204362+0000","last_fullsized":"2024-09-14T07:36:13.204362+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:46:17.306187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.210522+0000","last_change":"2024-09-14T06:52:02.055138+0000","last_active":"2024-09-14T07:36:13.210522+0000","last_peered":"2024-09-14T07:36:13.210522+0000","last_clean":"2024-09-14T07:36:13.210522+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T07:36:13.210522+0000","last_undegraded":"2024-09-14T07:36:13.210522+0000","last_fullsized":"2024-09-14T07:36:13.210522+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:01:29.208041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.193329+0000","last_change":"2024-09-14T06:52:02.053552+0000","last_active":"2024-09-14T07:36:13.193329+0000","last_peered":"2024-09-14T07:36:13.193329+0000","last_clean":"2024-09-14T07:36:13.193329+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T07:36:13.193329+0000","last_undegraded":"2024-09-14T07:36:13.193329+0000","last_fullsized":"2024-09-14T07:36:13.193329+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:07.068171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":837,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.298173+0000","last_change":"2024-09-14T06:52:02.063567+0000","last_active":"2024-09-14T07:36:13.298173+0000","last_peered":"2024-09-14T07:36:13.298173+0000","last_clean":"2024-09-14T07:36:13.298173+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T07:36:13.298173+0000","last_undegraded":"2024-09-14T07:36:13.298173+0000","last_fullsized":"2024-09-14T07:36:13.298173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:29.285798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.992860+0000","last_change":"2024-09-14T06:52:02.240204+0000","last_active":"2024-09-14T07:36:13.992860+0000","last_peered":"2024-09-14T07:36:13.992860+0000","last_clean":"2024-09-14T07:36:13.992860+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T07:36:13.992860+0000","last_undegraded":"2024-09-14T07:36:13.992860+0000","last_fullsized":"2024-09-14T07:36:13.992860+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:50.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":836,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.977367+0000","last_change":"2024-09-14T06:52:02.240272+0000","last_active":"2024-09-14T07:36:13.977367+0000","last_peered":"2024-09-14T07:36:13.977367+0000","last_clean":"2024-09-14T07:36:13.977367+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T07:36:13.977367+0000","last_undegraded":"2024-09-14T07:36:13.977367+0000","last_fullsized":"2024-09-14T07:36:13.977367+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:51:25.088755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":836,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.203438+0000","last_change":"2024-09-14T06:52:02.053793+0000","last_active":"2024-09-14T07:36:13.203438+0000","last_peered":"2024-09-14T07:36:13.203438+0000","last_clean":"2024-09-14T07:36:13.203438+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T07:36:13.203438+0000","last_undegraded":"2024-09-14T07:36:13.203438+0000","last_fullsized":"2024-09-14T07:36:13.203438+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:01:49.640848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":904,"reported_epoch":413,"state":"active+clean","last_fresh":"2024-09-14T07:36:13.197221+0000","last_change":"2024-09-14T06:51:56.116608+0000","last_active":"2024-09-14T07:36:13.197221+0000","last_peered":"2024-09-14T07:36:13.197221+0000","last_clean":"2024-09-14T07:36:13.197221+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T07:36:13.197221+0000","last_undegraded":"2024-09-14T07:36:13.197221+0000","last_fullsized":"2024-09-14T07:36:13.197221+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_clean_scrub_stamp":"2024-09-14T06:51:55.005796+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:26.973731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706201,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1530372,"kb_used_data":4784,"kb_used_omap":123,"kb_used_meta":1525444,"kb_avail":92841468,"statfs":{"total":96636764160,"available":95069663232,"internally_reserved":0,"allocated":4898816,"data_stored":2295681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":126718,"internal_metadata":1562054914},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706200,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1470156,"kb_used_data":5368,"kb_used_omap":128,"kb_used_meta":1464639,"kb_avail":92901684,"statfs":{"total":96636764160,"available":95131324416,"internally_reserved":0,"allocated":5496832,"data_stored":2886068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":131915,"internal_metadata":1499790517},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706200,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1199752,"kb_used_data":5364,"kb_used_omap":203,"kb_used_meta":1194164,"kb_avail":93172088,"statfs":{"total":96636764160,"available":95408218112,"internally_reserved":0,"allocated":5492736,"data_stored":2886049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208758,"internal_metadata":1222824074},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:36:17.611 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-14T07:36:17.906 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T07:36:17.907 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":414,"fsid":"798b1ebb-0651-4c92-aba2-17acca9548ca","created":"2024-09-14T06:51:43.417920+0000","modified":"2024-09-14T07:36:15.194739+0000","last_up_change":"2024-09-14T06:51:53.831394+0000","last_in_change":"2024-09-14T06:51:44.340418+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:54.956943+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T06:51:58.692382+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"82d699dd-3b35-4ab1-87a9-d69ce7d632eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6817","nonce":2226319647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6819","nonce":2226319647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6823","nonce":2226319647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":2226319647},{"type":"v1","addr":"172.21.15.169:6821","nonce":2226319647}]},"public_addr":"172.21.15.169:6817/2226319647","cluster_addr":"172.21.15.169:6819/2226319647","heartbeat_back_addr":"172.21.15.169:6823/2226319647","heartbeat_front_addr":"172.21.15.169:6821/2226319647","state":["exists","up"]},{"osd":1,"uuid":"0cf7e906-2a01-4c03-89a9-e4056eb15813","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6809","nonce":324375027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6811","nonce":324375027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6815","nonce":324375027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":324375027},{"type":"v1","addr":"172.21.15.169:6813","nonce":324375027}]},"public_addr":"172.21.15.169:6809/324375027","cluster_addr":"172.21.15.169:6811/324375027","heartbeat_back_addr":"172.21.15.169:6815/324375027","heartbeat_front_addr":"172.21.15.169:6813/324375027","state":["exists","up"]},{"osd":2,"uuid":"9dcade49-882f-40b8-b3a0-f14d759427c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6801","nonce":126512792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6803","nonce":126512792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6807","nonce":126512792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":126512792},{"type":"v1","addr":"172.21.15.169:6805","nonce":126512792}]},"public_addr":"172.21.15.169:6801/126512792","cluster_addr":"172.21.15.169:6803/126512792","heartbeat_back_addr":"172.21.15.169:6807/126512792","heartbeat_front_addr":"172.21.15.169:6805/126512792","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:51.167791+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.668476+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:50.467076+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.169:0/2585035416":"2024-09-14T08:34:50.201605+0000","172.21.15.169:0/2633628192":"2024-09-14T08:33:34.430341+0000","172.21.15.169:0/2742057021":"2024-09-14T08:33:00.522409+0000","172.21.15.169:0/3718911200":"2024-09-14T08:32:12.982770+0000","172.21.15.169:0/3967684340":"2024-09-14T08:31:30.330394+0000","172.21.15.169:0/2953961172":"2024-09-14T08:30:09.769449+0000","172.21.15.169:0/1395743010":"2024-09-14T08:28:17.239044+0000","172.21.15.169:0/1417395472":"2024-09-14T08:20:36.609222+0000","172.21.15.169:0/3130449864":"2024-09-14T08:25:56.648608+0000","172.21.15.169:0/3689643446":"2024-09-14T08:18:47.590374+0000","172.21.15.169:0/263220778":"2024-09-14T07:56:00.416125+0000","172.21.15.169:0/743624103":"2024-09-14T08:00:59.026651+0000","172.21.15.169:0/2930150881":"2024-09-14T08:28:08.903530+0000","172.21.15.169:0/1417879453":"2024-09-14T08:27:56.152987+0000","172.21.15.169:0/3663888791":"2024-09-14T08:18:19.215398+0000","172.21.15.169:0/3185553107":"2024-09-14T08:20:11.776531+0000","172.21.15.169:0/1368368094":"2024-09-14T08:12:03.554748+0000","172.21.15.169:0/1556648059":"2024-09-14T08:16:35.878828+0000","172.21.15.169:0/3398545639":"2024-09-14T07:54:29.246859+0000","172.21.15.169:0/1812874983":"2024-09-14T08:17:47.659846+0000","172.21.15.169:0/1599814985":"2024-09-14T08:17:08.908753+0000","172.21.15.169:0/3867463560":"2024-09-14T07:57:18.774580+0000","172.21.15.169:0/3510778663":"2024-09-14T08:25:13.019577+0000","172.21.15.169:0/4264491951":"2024-09-14T08:20:23.200041+0000","172.21.15.169:0/595296987":"2024-09-14T08:19:13.514900+0000","172.21.15.169:0/323761084":"2024-09-14T08:02:14.126275+0000","172.21.15.169:0/2832030136":"2024-09-14T08:12:50.819805+0000","172.21.15.169:0/1757489818":"2024-09-14T08:04:06.943201+0000","172.21.15.169:0/4277771155":"2024-09-14T08:22:13.736164+0000","172.21.15.169:0/3397649079":"2024-09-14T08:27:06.583778+0000","172.21.15.169:0/630577874":"2024-09-14T08:11:48.174400+0000","172.21.15.169:0/2607936539":"2024-09-14T08:22:56.312198+0000","172.21.15.169:0/1514291508":"2024-09-14T08:30:53.889656+0000","172.21.15.169:0/1061632546":"2024-09-14T08:14:55.922601+0000","172.21.15.169:0/3016733757":"2024-09-14T08:19:02.343894+0000","172.21.15.169:0/3520429646":"2024-09-14T08:17:02.316756+0000","172.21.15.169:0/1471585250":"2024-09-14T07:59:35.406665+0000","172.21.15.169:0/3827494442":"2024-09-14T07:58:39.990925+0000","172.21.15.169:0/4216142954":"2024-09-14T08:11:16.674663+0000","172.21.15.169:0/3402760618":"2024-09-14T08:05:31.885275+0000","172.21.15.169:0/2420096927":"2024-09-14T08:03:14.824235+0000","172.21.15.169:0/309716379":"2024-09-14T08:22:39.148682+0000","172.21.15.169:0/3302042426":"2024-09-14T08:27:27.605331+0000","172.21.15.169:0/1087977727":"2024-09-14T08:36:10.671524+0000","172.21.15.169:0/3113630275":"2024-09-14T08:20:52.391059+0000","172.21.15.169:0/3595464051":"2024-09-14T08:10:45.276109+0000","172.21.15.169:0/2201646599":"2024-09-14T08:25:26.619346+0000","172.21.15.169:0/3770209400":"2024-09-14T08:20:59.841986+0000","172.21.15.169:0/1007781535":"2024-09-14T08:26:21.112048+0000","172.21.15.169:0/154082109":"2024-09-14T08:23:47.308780+0000","172.21.15.169:0/1729833800":"2024-09-14T08:21:21.783385+0000","172.21.15.169:0/997706697":"2024-09-14T08:23:27.352113+0000","172.21.15.169:0/367387070":"2024-09-14T08:21:43.330309+0000","172.21.15.169:0/3416212748":"2024-09-14T08:34:12.557520+0000","172.21.15.169:0/2443698242":"2024-09-14T08:21:58.793751+0000","172.21.15.169:0/1808543247":"2024-09-14T08:22:20.773272+0000","172.21.15.169:0/2887087256":"2024-09-14T08:23:35.691257+0000","172.21.15.169:0/1746121554":"2024-09-14T08:23:05.563133+0000","172.21.15.169:0/3414303966":"2024-09-14T08:23:16.937426+0000","172.21.15.169:0/3448802553":"2024-09-14T08:23:58.810824+0000","172.21.15.169:0/3464154351":"2024-09-14T08:24:48.035281+0000","172.21.15.169:0/3775873571":"2024-09-14T08:18:32.675528+0000","172.21.15.169:0/944998265":"2024-09-14T08:19:58.759145+0000","172.21.15.169:0/3721478540":"2024-09-14T08:25:06.447910+0000","172.21.15.169:0/3618106776":"2024-09-14T08:25:35.898251+0000","172.21.15.169:0/3342304422":"2024-09-14T08:25:43.973425+0000","172.21.15.169:0/3873208652":"2024-09-14T08:35:32.409828+0000","172.21.15.169:0/3378000178":"2024-09-14T08:26:09.784968+0000","172.21.15.169:0/438228671":"2024-09-14T08:22:46.485402+0000","172.21.15.169:0/1649736411":"2024-09-14T08:27:36.692642+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T07:36:18.925 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2024-09-14T07:36:18.925 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-14T07:36:19.086 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2024-09-14T07:36:19.086 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T07:36:19.086 INFO:teuthology.orchestra.run.smithi169.stdout:} 2024-09-14T07:36:19.102 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2024-09-14T07:36:19.392 INFO:teuthology.orchestra.run.smithi169.stderr:instructed osd(s) 0 to deep-scrub 2024-09-14T07:36:19.410 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2024-09-14T07:36:19.410 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-14T07:36:19.540 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2024-09-14T07:36:19.540 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T07:36:19.540 INFO:teuthology.orchestra.run.smithi169.stdout:} 2024-09-14T07:36:19.556 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2024-09-14T07:36:19.833 INFO:teuthology.orchestra.run.smithi169.stderr:instructed osd(s) 1 to deep-scrub 2024-09-14T07:36:19.851 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2024-09-14T07:36:19.851 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-14T07:36:19.983 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2024-09-14T07:36:19.983 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T07:36:19.983 INFO:teuthology.orchestra.run.smithi169.stdout:} 2024-09-14T07:36:19.998 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2024-09-14T07:36:20.280 INFO:teuthology.orchestra.run.smithi169.stderr:instructed osd(s) 2 to deep-scrub 2024-09-14T07:36:20.298 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T07:36:20.596 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T07:36:20.597 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T07:36:20.616 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":1745,"stamp":"2024-09-14T07:36:19.919721+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":130,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4236420,"kb_used_data":15556,"kb_used_omap":456,"kb_used_meta":4220343,"kb_avail":278879100,"statfs":{"total":289910292480,"available":285572198400,"internally_reserved":0,"allocated":15929344,"data_stored":8087680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":467391,"internal_metadata":4321631809},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504383574,"num_objects":-168,"num_object_clones":-1,"num_object_copies":-336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-168,"num_whiteouts":-1,"num_read":-826418,"num_read_kb":-48226183,"num_write":-647404,"num_write_kb":-22868151,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001898"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":836,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.599132+0000","last_change":"2024-09-14T06:52:02.054336+0000","last_active":"2024-09-14T07:36:15.599132+0000","last_peered":"2024-09-14T07:36:15.599132+0000","last_clean":"2024-09-14T07:36:15.599132+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T07:36:15.599132+0000","last_undegraded":"2024-09-14T07:36:15.599132+0000","last_fullsized":"2024-09-14T07:36:15.599132+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:38:29.413964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":836,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.598422+0000","last_change":"2024-09-14T06:52:02.054459+0000","last_active":"2024-09-14T07:36:15.598422+0000","last_peered":"2024-09-14T07:36:15.598422+0000","last_clean":"2024-09-14T07:36:15.598422+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T07:36:15.598422+0000","last_undegraded":"2024-09-14T07:36:15.598422+0000","last_fullsized":"2024-09-14T07:36:15.598422+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:46:17.306187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":837,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.603527+0000","last_change":"2024-09-14T06:52:02.055138+0000","last_active":"2024-09-14T07:36:15.603527+0000","last_peered":"2024-09-14T07:36:15.603527+0000","last_clean":"2024-09-14T07:36:15.603527+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T07:36:15.603527+0000","last_undegraded":"2024-09-14T07:36:15.603527+0000","last_fullsized":"2024-09-14T07:36:15.603527+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:01:29.208041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":837,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.600051+0000","last_change":"2024-09-14T06:52:02.053552+0000","last_active":"2024-09-14T07:36:15.600051+0000","last_peered":"2024-09-14T07:36:15.600051+0000","last_clean":"2024-09-14T07:36:15.600051+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T07:36:15.600051+0000","last_undegraded":"2024-09-14T07:36:15.600051+0000","last_fullsized":"2024-09-14T07:36:15.600051+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:18:07.068171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":839,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.678921+0000","last_change":"2024-09-14T06:52:02.063567+0000","last_active":"2024-09-14T07:36:15.678921+0000","last_peered":"2024-09-14T07:36:15.678921+0000","last_clean":"2024-09-14T07:36:15.678921+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T07:36:15.678921+0000","last_undegraded":"2024-09-14T07:36:15.678921+0000","last_fullsized":"2024-09-14T07:36:15.678921+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:38:29.285798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":837,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.209445+0000","last_change":"2024-09-14T06:52:02.240204+0000","last_active":"2024-09-14T07:36:15.209445+0000","last_peered":"2024-09-14T07:36:15.209445+0000","last_clean":"2024-09-14T07:36:15.209445+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T07:36:15.209445+0000","last_undegraded":"2024-09-14T07:36:15.209445+0000","last_fullsized":"2024-09-14T07:36:15.209445+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:50.128762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":838,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.204809+0000","last_change":"2024-09-14T06:52:02.240272+0000","last_active":"2024-09-14T07:36:15.204809+0000","last_peered":"2024-09-14T07:36:15.204809+0000","last_clean":"2024-09-14T07:36:15.204809+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T07:36:15.204809+0000","last_undegraded":"2024-09-14T07:36:15.204809+0000","last_fullsized":"2024-09-14T07:36:15.204809+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:51:25.088755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":838,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.601341+0000","last_change":"2024-09-14T06:52:02.053793+0000","last_active":"2024-09-14T07:36:15.601341+0000","last_peered":"2024-09-14T07:36:15.601341+0000","last_clean":"2024-09-14T07:36:15.601341+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T07:36:15.601341+0000","last_undegraded":"2024-09-14T07:36:15.601341+0000","last_fullsized":"2024-09-14T07:36:15.601341+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:59.024318+0000","last_clean_scrub_stamp":"2024-09-14T06:51:59.024318+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:01:49.640848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":906,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:15.598384+0000","last_change":"2024-09-14T06:51:56.116608+0000","last_active":"2024-09-14T07:36:15.598384+0000","last_peered":"2024-09-14T07:36:15.598384+0000","last_clean":"2024-09-14T07:36:15.598384+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T07:36:15.598384+0000","last_undegraded":"2024-09-14T07:36:15.598384+0000","last_fullsized":"2024-09-14T07:36:15.598384+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:55.005796+0000","last_clean_scrub_stamp":"2024-09-14T06:51:55.005796+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:13:26.973731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706202,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1548436,"kb_used_data":4796,"kb_used_omap":123,"kb_used_meta":1543492,"kb_avail":92823404,"statfs":{"total":96636764160,"available":95051165696,"internally_reserved":0,"allocated":4911104,"data_stored":2302302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":126718,"internal_metadata":1580536066},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706201,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1470168,"kb_used_data":5380,"kb_used_omap":128,"kb_used_meta":1464639,"kb_avail":92901672,"statfs":{"total":96636764160,"available":95131312128,"internally_reserved":0,"allocated":5509120,"data_stored":2892689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":131915,"internal_metadata":1499790517},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706201,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1217816,"kb_used_data":5380,"kb_used_omap":203,"kb_used_meta":1212212,"kb_avail":93154024,"statfs":{"total":96636764160,"available":95389720576,"internally_reserved":0,"allocated":5509120,"data_stored":2892689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208758,"internal_metadata":1241305226},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:36:20.620 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.620 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.620 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-14T06:51:59.024318+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=59, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-14T06:51:55.005796+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=55, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=36, tm_sec=17, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:36:20.621 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-14T07:36:40.622 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-14T07:36:40.937 INFO:teuthology.orchestra.run.smithi169.stdout: 2024-09-14T07:36:40.937 INFO:teuthology.orchestra.run.smithi169.stderr:dumped all 2024-09-14T07:36:40.953 INFO:teuthology.orchestra.run.smithi169.stdout:{"pg_ready":true,"pg_map":{"version":1755,"stamp":"2024-09-14T07:36:39.924726+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59,"ondisk_log_size":59,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4236404,"kb_used_data":15556,"kb_used_omap":456,"kb_used_meta":4220343,"kb_avail":278879116,"statfs":{"total":289910292480,"available":285572214784,"internally_reserved":0,"allocated":15929344,"data_stored":8087680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":467391,"internal_metadata":4321631809},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003044"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":844,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:25.578402+0000","last_change":"2024-09-14T07:36:25.578402+0000","last_active":"2024-09-14T07:36:25.578402+0000","last_peered":"2024-09-14T07:36:25.578402+0000","last_clean":"2024-09-14T07:36:25.578402+0000","last_became_active":"2024-09-14T06:52:00.052353+0000","last_became_peered":"2024-09-14T06:52:00.052353+0000","last_unstale":"2024-09-14T07:36:25.578402+0000","last_undegraded":"2024-09-14T07:36:25.578402+0000","last_fullsized":"2024-09-14T07:36:25.578402+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:36:25.578215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:36:25.578215+0000","last_clean_scrub_stamp":"2024-09-14T07:36:25.578215+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:40.339317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020233880000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":844,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:24.582935+0000","last_change":"2024-09-14T07:36:24.582935+0000","last_active":"2024-09-14T07:36:24.582935+0000","last_peered":"2024-09-14T07:36:24.582935+0000","last_clean":"2024-09-14T07:36:24.582935+0000","last_became_active":"2024-09-14T06:52:00.051663+0000","last_became_peered":"2024-09-14T06:52:00.051663+0000","last_unstale":"2024-09-14T07:36:24.582935+0000","last_undegraded":"2024-09-14T07:36:24.582935+0000","last_fullsized":"2024-09-14T07:36:24.582935+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:36:24.582759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:36:24.582759+0000","last_clean_scrub_stamp":"2024-09-14T07:36:24.582759+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:20.139108+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0019868049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":845,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:23.567779+0000","last_change":"2024-09-14T07:36:23.567779+0000","last_active":"2024-09-14T07:36:23.567779+0000","last_peered":"2024-09-14T07:36:23.567779+0000","last_clean":"2024-09-14T07:36:23.567779+0000","last_became_active":"2024-09-14T06:52:00.051664+0000","last_became_peered":"2024-09-14T06:52:00.051664+0000","last_unstale":"2024-09-14T07:36:23.567779+0000","last_undegraded":"2024-09-14T07:36:23.567779+0000","last_fullsized":"2024-09-14T07:36:23.567779+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:36:23.567597+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:36:23.567597+0000","last_clean_scrub_stamp":"2024-09-14T07:36:23.567597+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:39:27.239359+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.002550947,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":845,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:21.598463+0000","last_change":"2024-09-14T07:36:21.598463+0000","last_active":"2024-09-14T07:36:21.598463+0000","last_peered":"2024-09-14T07:36:21.598463+0000","last_clean":"2024-09-14T07:36:21.598463+0000","last_became_active":"2024-09-14T06:52:00.049088+0000","last_became_peered":"2024-09-14T06:52:00.049088+0000","last_unstale":"2024-09-14T07:36:21.598463+0000","last_undegraded":"2024-09-14T07:36:21.598463+0000","last_fullsized":"2024-09-14T07:36:21.598463+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:36:21.598263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:36:21.598263+0000","last_clean_scrub_stamp":"2024-09-14T07:36:21.598263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:31.702454+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0013181930000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"414'3","reported_seq":848,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:19.549305+0000","last_change":"2024-09-14T07:36:19.549305+0000","last_active":"2024-09-14T07:36:19.549305+0000","last_peered":"2024-09-14T07:36:19.549305+0000","last_clean":"2024-09-14T07:36:19.549305+0000","last_became_active":"2024-09-14T06:52:00.050179+0000","last_became_peered":"2024-09-14T06:52:00.050179+0000","last_unstale":"2024-09-14T07:36:19.549305+0000","last_undegraded":"2024-09-14T07:36:19.549305+0000","last_fullsized":"2024-09-14T07:36:19.549305+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'3","last_scrub_stamp":"2024-09-14T07:36:19.549226+0000","last_deep_scrub":"414'3","last_deep_scrub_stamp":"2024-09-14T07:36:19.549226+0000","last_clean_scrub_stamp":"2024-09-14T07:36:19.549226+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:19:34.157796+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.002486491,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":845,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:22.169957+0000","last_change":"2024-09-14T07:36:22.169957+0000","last_active":"2024-09-14T07:36:22.169957+0000","last_peered":"2024-09-14T07:36:22.169957+0000","last_clean":"2024-09-14T07:36:22.169957+0000","last_became_active":"2024-09-14T06:52:00.050787+0000","last_became_peered":"2024-09-14T06:52:00.050787+0000","last_unstale":"2024-09-14T07:36:22.169957+0000","last_undegraded":"2024-09-14T07:36:22.169957+0000","last_fullsized":"2024-09-14T07:36:22.169957+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:36:22.169772+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:36:22.169772+0000","last_clean_scrub_stamp":"2024-09-14T07:36:22.169772+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:35:49.622953+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00043176800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":846,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:21.215176+0000","last_change":"2024-09-14T07:36:21.215176+0000","last_active":"2024-09-14T07:36:21.215176+0000","last_peered":"2024-09-14T07:36:21.215176+0000","last_clean":"2024-09-14T07:36:21.215176+0000","last_became_active":"2024-09-14T06:52:00.051047+0000","last_became_peered":"2024-09-14T06:52:00.051047+0000","last_unstale":"2024-09-14T07:36:21.215176+0000","last_undegraded":"2024-09-14T07:36:21.215176+0000","last_fullsized":"2024-09-14T07:36:21.215176+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:36:21.214982+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:36:21.214982+0000","last_clean_scrub_stamp":"2024-09-14T07:36:21.214982+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:33:15.439712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049403100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"414'2","reported_seq":847,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:20.606118+0000","last_change":"2024-09-14T07:36:20.606118+0000","last_active":"2024-09-14T07:36:20.606118+0000","last_peered":"2024-09-14T07:36:20.606118+0000","last_clean":"2024-09-14T07:36:20.606118+0000","last_became_active":"2024-09-14T06:52:00.046992+0000","last_became_peered":"2024-09-14T06:52:00.046992+0000","last_unstale":"2024-09-14T07:36:20.606118+0000","last_undegraded":"2024-09-14T07:36:20.606118+0000","last_fullsized":"2024-09-14T07:36:20.606118+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'2","last_scrub_stamp":"2024-09-14T07:36:20.606021+0000","last_deep_scrub":"414'2","last_deep_scrub_stamp":"2024-09-14T07:36:20.606021+0000","last_clean_scrub_stamp":"2024-09-14T07:36:20.606021+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:56.233499+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.001504102,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"414'54","reported_seq":916,"reported_epoch":414,"state":"active+clean","last_fresh":"2024-09-14T07:36:22.602134+0000","last_change":"2024-09-14T07:36:22.602134+0000","last_active":"2024-09-14T07:36:22.602134+0000","last_peered":"2024-09-14T07:36:22.602134+0000","last_clean":"2024-09-14T07:36:22.602134+0000","last_became_active":"2024-09-14T06:51:56.114132+0000","last_became_peered":"2024-09-14T06:51:56.114132+0000","last_unstale":"2024-09-14T07:36:22.602134+0000","last_undegraded":"2024-09-14T07:36:22.602134+0000","last_fullsized":"2024-09-14T07:36:22.602134+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'54","last_scrub_stamp":"2024-09-14T07:36:22.601929+0000","last_deep_scrub":"414'54","last_deep_scrub_stamp":"2024-09-14T07:36:22.601929+0000","last_clean_scrub_stamp":"2024-09-14T07:36:22.601929+0000","objects_scrubbed":2,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.614168+0000","scrub_duration":11,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54,"ondisk_log_size":54,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706206,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1548428,"kb_used_data":4796,"kb_used_omap":123,"kb_used_meta":1543492,"kb_avail":92823412,"statfs":{"total":96636764160,"available":95051173888,"internally_reserved":0,"allocated":4911104,"data_stored":2302302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":126718,"internal_metadata":1580536066},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706205,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1470164,"kb_used_data":5380,"kb_used_omap":128,"kb_used_meta":1464639,"kb_avail":92901676,"statfs":{"total":96636764160,"available":95131316224,"internally_reserved":0,"allocated":5509120,"data_stored":2892689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":131915,"internal_metadata":1499790517},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706205,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1217812,"kb_used_data":5380,"kb_used_omap":203,"kb_used_meta":1212212,"kb_avail":93154028,"statfs":{"total":96636764160,"available":95389724672,"internally_reserved":0,"allocated":5509120,"data_stored":2892689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208758,"internal_metadata":1241305226},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:36:40.955 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2024-09-14T07:36:41.275 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-14T07:36:41.275 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-14T07:36:41.275 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2024-09-14T07:36:41.275 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:43.156 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2024-09-14T07:36:47.377 INFO:tasks.ceph.cluster2.osd.0:Stopped 2024-09-14T07:36:47.377 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2024-09-14T07:36:47.378 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:47.477 INFO:tasks.ceph.cluster2.osd.1:Stopped 2024-09-14T07:36:47.478 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2024-09-14T07:36:47.478 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:48.359 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~0s 2024-09-14T07:36:48.359 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 has been restored 2024-09-14T07:36:53.561 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~5s 2024-09-14T07:36:53.579 INFO:tasks.ceph.cluster2.osd.2:Stopped 2024-09-14T07:36:53.579 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-14T07:36:53.579 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2024-09-14T07:36:53.579 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:58.663 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.mgr.x is failed for ~0s 2024-09-14T07:36:58.663 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 has been restored 2024-09-14T07:36:59.681 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2024-09-14T07:36:59.682 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-14T07:36:59.682 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2024-09-14T07:36:59.682 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:36:59.708 INFO:tasks.ceph.cluster2.mon.a:Stopped 2024-09-14T07:36:59.709 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-14T07:36:59.709 DEBUG:teuthology.orchestra.run.smithi169:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-14T07:37:00.098 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T07:37:00.098 DEBUG:teuthology.orchestra.run.smithi169:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2024-09-14T07:37:00.480 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T07:37:00.481 DEBUG:teuthology.orchestra.run.smithi169:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2024-09-14T07:37:00.811 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T07:37:00.811 DEBUG:teuthology.orchestra.run.smithi169:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2024-09-14T07:37:01.194 INFO:tasks.ceph:Archiving mon data... 2024-09-14T07:37:01.196 DEBUG:teuthology.misc:Transferring archived files from smithi169:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/data/cluster2.mon.a.tgz 2024-09-14T07:37:01.196 DEBUG:teuthology.orchestra.run.smithi169:> mktemp 2024-09-14T07:37:01.201 INFO:teuthology.orchestra.run.smithi169.stdout:/tmp/tmp.mvJtoUfNzb 2024-09-14T07:37:01.201 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.mvJtoUfNzb 2024-09-14T07:37:01.374 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0666 /tmp/tmp.mvJtoUfNzb 2024-09-14T07:37:01.461 DEBUG:teuthology.orchestra.remote:smithi169:/tmp/tmp.mvJtoUfNzb is 455KB 2024-09-14T07:37:01.526 DEBUG:teuthology.orchestra.run.smithi169:> rm -fr /tmp/tmp.mvJtoUfNzb 2024-09-14T07:37:01.531 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-14T07:37:01.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T07:37:01.534 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T07:37:01.795 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-14T07:37:01.809 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-14T07:37:01.809 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T07:37:02.343 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T07:37:02.343 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T07:37:02.385 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":1705,"stamp":"2024-09-14T07:37:01.976004+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3818704,"kb_used_data":9232,"kb_used_omap":438,"kb_used_meta":3808969,"kb_avail":279296816,"statfs":{"total":289910292480,"available":285999939584,"internally_reserved":0,"allocated":9453568,"data_stored":5540101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":448962,"internal_metadata":3900384830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002999"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.731806+0000","last_change":"2024-09-14T06:51:23.124440+0000","last_active":"2024-09-14T07:36:14.731806+0000","last_peered":"2024-09-14T07:36:14.731806+0000","last_clean":"2024-09-14T07:36:14.731806+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T07:36:14.731806+0000","last_undegraded":"2024-09-14T07:36:14.731806+0000","last_fullsized":"2024-09-14T07:36:14.731806+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:54.063248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736557+0000","last_change":"2024-09-14T06:51:23.124277+0000","last_active":"2024-09-14T07:36:14.736557+0000","last_peered":"2024-09-14T07:36:14.736557+0000","last_clean":"2024-09-14T07:36:14.736557+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T07:36:14.736557+0000","last_undegraded":"2024-09-14T07:36:14.736557+0000","last_fullsized":"2024-09-14T07:36:14.736557+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:36:43.389449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.729160+0000","last_change":"2024-09-14T06:51:23.123529+0000","last_active":"2024-09-14T07:36:14.729160+0000","last_peered":"2024-09-14T07:36:14.729160+0000","last_clean":"2024-09-14T07:36:14.729160+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T07:36:14.729160+0000","last_undegraded":"2024-09-14T07:36:14.729160+0000","last_fullsized":"2024-09-14T07:36:14.729160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:18.274940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736272+0000","last_change":"2024-09-14T06:51:23.123532+0000","last_active":"2024-09-14T07:36:14.736272+0000","last_peered":"2024-09-14T07:36:14.736272+0000","last_clean":"2024-09-14T07:36:14.736272+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T07:36:14.736272+0000","last_undegraded":"2024-09-14T07:36:14.736272+0000","last_fullsized":"2024-09-14T07:36:14.736272+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:21.958324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.453786+0000","last_change":"2024-09-14T06:51:23.118966+0000","last_active":"2024-09-14T07:36:17.453786+0000","last_peered":"2024-09-14T07:36:17.453786+0000","last_clean":"2024-09-14T07:36:17.453786+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T07:36:17.453786+0000","last_undegraded":"2024-09-14T07:36:17.453786+0000","last_fullsized":"2024-09-14T07:36:17.453786+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:59.628972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484984+0000","last_change":"2024-09-14T06:51:23.113503+0000","last_active":"2024-09-14T07:36:17.484984+0000","last_peered":"2024-09-14T07:36:17.484984+0000","last_clean":"2024-09-14T07:36:17.484984+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T07:36:17.484984+0000","last_undegraded":"2024-09-14T07:36:17.484984+0000","last_fullsized":"2024-09-14T07:36:17.484984+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:15.559943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484900+0000","last_change":"2024-09-14T06:51:23.113565+0000","last_active":"2024-09-14T07:36:17.484900+0000","last_peered":"2024-09-14T07:36:17.484900+0000","last_clean":"2024-09-14T07:36:17.484900+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T07:36:17.484900+0000","last_undegraded":"2024-09-14T07:36:17.484900+0000","last_fullsized":"2024-09-14T07:36:17.484900+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:39.862806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.734420+0000","last_change":"2024-09-14T06:51:23.123208+0000","last_active":"2024-09-14T07:36:14.734420+0000","last_peered":"2024-09-14T07:36:14.734420+0000","last_clean":"2024-09-14T07:36:14.734420+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T07:36:14.734420+0000","last_undegraded":"2024-09-14T07:36:14.734420+0000","last_fullsized":"2024-09-14T07:36:14.734420+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:58.546304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":748,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.725176+0000","last_change":"2024-09-14T06:51:17.941930+0000","last_active":"2024-09-14T07:36:14.725176+0000","last_peered":"2024-09-14T07:36:14.725176+0000","last_clean":"2024-09-14T07:36:14.725176+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T07:36:14.725176+0000","last_undegraded":"2024-09-14T07:36:14.725176+0000","last_fullsized":"2024-09-14T07:36:14.725176+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_clean_scrub_stamp":"2024-09-14T06:51:16.017137+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:40.900521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738925,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":703124,"kb_used_data":2688,"kb_used_omap":149,"kb_used_meta":700266,"kb_avail":93668716,"statfs":{"total":96636764160,"available":95916765184,"internally_reserved":0,"allocated":2752512,"data_stored":1453109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":153404,"internal_metadata":717072580},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738921,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1405344,"kb_used_data":3272,"kb_used_omap":137,"kb_used_meta":1401910,"kb_avail":92966496,"statfs":{"total":96636764160,"available":95197691904,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141048,"internal_metadata":1435556104},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738921,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1710236,"kb_used_data":3272,"kb_used_omap":150,"kb_used_meta":1706793,"kb_avail":92661604,"statfs":{"total":96636764160,"available":94885482496,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":154510,"internal_metadata":1747756146},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:37:02.386 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T07:37:02.677 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T07:37:02.677 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T07:37:02.694 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":1705,"stamp":"2024-09-14T07:37:01.976004+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3818704,"kb_used_data":9232,"kb_used_omap":438,"kb_used_meta":3808969,"kb_avail":279296816,"statfs":{"total":289910292480,"available":285999939584,"internally_reserved":0,"allocated":9453568,"data_stored":5540101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":448962,"internal_metadata":3900384830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002999"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.731806+0000","last_change":"2024-09-14T06:51:23.124440+0000","last_active":"2024-09-14T07:36:14.731806+0000","last_peered":"2024-09-14T07:36:14.731806+0000","last_clean":"2024-09-14T07:36:14.731806+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T07:36:14.731806+0000","last_undegraded":"2024-09-14T07:36:14.731806+0000","last_fullsized":"2024-09-14T07:36:14.731806+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:54.063248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736557+0000","last_change":"2024-09-14T06:51:23.124277+0000","last_active":"2024-09-14T07:36:14.736557+0000","last_peered":"2024-09-14T07:36:14.736557+0000","last_clean":"2024-09-14T07:36:14.736557+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T07:36:14.736557+0000","last_undegraded":"2024-09-14T07:36:14.736557+0000","last_fullsized":"2024-09-14T07:36:14.736557+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:36:43.389449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.729160+0000","last_change":"2024-09-14T06:51:23.123529+0000","last_active":"2024-09-14T07:36:14.729160+0000","last_peered":"2024-09-14T07:36:14.729160+0000","last_clean":"2024-09-14T07:36:14.729160+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T07:36:14.729160+0000","last_undegraded":"2024-09-14T07:36:14.729160+0000","last_fullsized":"2024-09-14T07:36:14.729160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:18.274940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736272+0000","last_change":"2024-09-14T06:51:23.123532+0000","last_active":"2024-09-14T07:36:14.736272+0000","last_peered":"2024-09-14T07:36:14.736272+0000","last_clean":"2024-09-14T07:36:14.736272+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T07:36:14.736272+0000","last_undegraded":"2024-09-14T07:36:14.736272+0000","last_fullsized":"2024-09-14T07:36:14.736272+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:21.958324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.453786+0000","last_change":"2024-09-14T06:51:23.118966+0000","last_active":"2024-09-14T07:36:17.453786+0000","last_peered":"2024-09-14T07:36:17.453786+0000","last_clean":"2024-09-14T07:36:17.453786+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T07:36:17.453786+0000","last_undegraded":"2024-09-14T07:36:17.453786+0000","last_fullsized":"2024-09-14T07:36:17.453786+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:59.628972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484984+0000","last_change":"2024-09-14T06:51:23.113503+0000","last_active":"2024-09-14T07:36:17.484984+0000","last_peered":"2024-09-14T07:36:17.484984+0000","last_clean":"2024-09-14T07:36:17.484984+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T07:36:17.484984+0000","last_undegraded":"2024-09-14T07:36:17.484984+0000","last_fullsized":"2024-09-14T07:36:17.484984+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:15.559943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484900+0000","last_change":"2024-09-14T06:51:23.113565+0000","last_active":"2024-09-14T07:36:17.484900+0000","last_peered":"2024-09-14T07:36:17.484900+0000","last_clean":"2024-09-14T07:36:17.484900+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T07:36:17.484900+0000","last_undegraded":"2024-09-14T07:36:17.484900+0000","last_fullsized":"2024-09-14T07:36:17.484900+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:39.862806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.734420+0000","last_change":"2024-09-14T06:51:23.123208+0000","last_active":"2024-09-14T07:36:14.734420+0000","last_peered":"2024-09-14T07:36:14.734420+0000","last_clean":"2024-09-14T07:36:14.734420+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T07:36:14.734420+0000","last_undegraded":"2024-09-14T07:36:14.734420+0000","last_fullsized":"2024-09-14T07:36:14.734420+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:58.546304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":748,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.725176+0000","last_change":"2024-09-14T06:51:17.941930+0000","last_active":"2024-09-14T07:36:14.725176+0000","last_peered":"2024-09-14T07:36:14.725176+0000","last_clean":"2024-09-14T07:36:14.725176+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T07:36:14.725176+0000","last_undegraded":"2024-09-14T07:36:14.725176+0000","last_fullsized":"2024-09-14T07:36:14.725176+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_clean_scrub_stamp":"2024-09-14T06:51:16.017137+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:40.900521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738925,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":703124,"kb_used_data":2688,"kb_used_omap":149,"kb_used_meta":700266,"kb_avail":93668716,"statfs":{"total":96636764160,"available":95916765184,"internally_reserved":0,"allocated":2752512,"data_stored":1453109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":153404,"internal_metadata":717072580},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738921,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1405344,"kb_used_data":3272,"kb_used_omap":137,"kb_used_meta":1401910,"kb_avail":92966496,"statfs":{"total":96636764160,"available":95197691904,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141048,"internal_metadata":1435556104},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738921,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1710236,"kb_used_data":3272,"kb_used_omap":150,"kb_used_meta":1706793,"kb_avail":92661604,"statfs":{"total":96636764160,"available":94885482496,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":154510,"internal_metadata":1747756146},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:37:02.695 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-14T07:37:02.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T07:37:02.983 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T07:37:02.983 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T07:37:03.000 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":1705,"stamp":"2024-09-14T07:37:01.976004+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3818704,"kb_used_data":9232,"kb_used_omap":438,"kb_used_meta":3808969,"kb_avail":279296816,"statfs":{"total":289910292480,"available":285999939584,"internally_reserved":0,"allocated":9453568,"data_stored":5540101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":448962,"internal_metadata":3900384830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002999"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.731806+0000","last_change":"2024-09-14T06:51:23.124440+0000","last_active":"2024-09-14T07:36:14.731806+0000","last_peered":"2024-09-14T07:36:14.731806+0000","last_clean":"2024-09-14T07:36:14.731806+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T07:36:14.731806+0000","last_undegraded":"2024-09-14T07:36:14.731806+0000","last_fullsized":"2024-09-14T07:36:14.731806+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:54.063248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736557+0000","last_change":"2024-09-14T06:51:23.124277+0000","last_active":"2024-09-14T07:36:14.736557+0000","last_peered":"2024-09-14T07:36:14.736557+0000","last_clean":"2024-09-14T07:36:14.736557+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T07:36:14.736557+0000","last_undegraded":"2024-09-14T07:36:14.736557+0000","last_fullsized":"2024-09-14T07:36:14.736557+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:36:43.389449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.729160+0000","last_change":"2024-09-14T06:51:23.123529+0000","last_active":"2024-09-14T07:36:14.729160+0000","last_peered":"2024-09-14T07:36:14.729160+0000","last_clean":"2024-09-14T07:36:14.729160+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T07:36:14.729160+0000","last_undegraded":"2024-09-14T07:36:14.729160+0000","last_fullsized":"2024-09-14T07:36:14.729160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:18.274940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736272+0000","last_change":"2024-09-14T06:51:23.123532+0000","last_active":"2024-09-14T07:36:14.736272+0000","last_peered":"2024-09-14T07:36:14.736272+0000","last_clean":"2024-09-14T07:36:14.736272+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T07:36:14.736272+0000","last_undegraded":"2024-09-14T07:36:14.736272+0000","last_fullsized":"2024-09-14T07:36:14.736272+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:21.958324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.453786+0000","last_change":"2024-09-14T06:51:23.118966+0000","last_active":"2024-09-14T07:36:17.453786+0000","last_peered":"2024-09-14T07:36:17.453786+0000","last_clean":"2024-09-14T07:36:17.453786+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T07:36:17.453786+0000","last_undegraded":"2024-09-14T07:36:17.453786+0000","last_fullsized":"2024-09-14T07:36:17.453786+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:59.628972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484984+0000","last_change":"2024-09-14T06:51:23.113503+0000","last_active":"2024-09-14T07:36:17.484984+0000","last_peered":"2024-09-14T07:36:17.484984+0000","last_clean":"2024-09-14T07:36:17.484984+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T07:36:17.484984+0000","last_undegraded":"2024-09-14T07:36:17.484984+0000","last_fullsized":"2024-09-14T07:36:17.484984+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:15.559943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484900+0000","last_change":"2024-09-14T06:51:23.113565+0000","last_active":"2024-09-14T07:36:17.484900+0000","last_peered":"2024-09-14T07:36:17.484900+0000","last_clean":"2024-09-14T07:36:17.484900+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T07:36:17.484900+0000","last_undegraded":"2024-09-14T07:36:17.484900+0000","last_fullsized":"2024-09-14T07:36:17.484900+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:39.862806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.734420+0000","last_change":"2024-09-14T06:51:23.123208+0000","last_active":"2024-09-14T07:36:14.734420+0000","last_peered":"2024-09-14T07:36:14.734420+0000","last_clean":"2024-09-14T07:36:14.734420+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T07:36:14.734420+0000","last_undegraded":"2024-09-14T07:36:14.734420+0000","last_fullsized":"2024-09-14T07:36:14.734420+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:58.546304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":748,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.725176+0000","last_change":"2024-09-14T06:51:17.941930+0000","last_active":"2024-09-14T07:36:14.725176+0000","last_peered":"2024-09-14T07:36:14.725176+0000","last_clean":"2024-09-14T07:36:14.725176+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T07:36:14.725176+0000","last_undegraded":"2024-09-14T07:36:14.725176+0000","last_fullsized":"2024-09-14T07:36:14.725176+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_clean_scrub_stamp":"2024-09-14T06:51:16.017137+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:40.900521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738925,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":703124,"kb_used_data":2688,"kb_used_omap":149,"kb_used_meta":700266,"kb_avail":93668716,"statfs":{"total":96636764160,"available":95916765184,"internally_reserved":0,"allocated":2752512,"data_stored":1453109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":153404,"internal_metadata":717072580},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738921,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1405344,"kb_used_data":3272,"kb_used_omap":137,"kb_used_meta":1401910,"kb_avail":92966496,"statfs":{"total":96636764160,"available":95197691904,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141048,"internal_metadata":1435556104},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738921,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1710236,"kb_used_data":3272,"kb_used_omap":150,"kb_used_meta":1706793,"kb_avail":92661604,"statfs":{"total":96636764160,"available":94885482496,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":154510,"internal_metadata":1747756146},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:37:03.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-14T07:37:03.278 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T07:37:03.278 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":334,"fsid":"95676e5b-ca8a-4e3d-9640-d6c7d09c88f4","created":"2024-09-14T06:51:04.184127+0000","modified":"2024-09-14T07:36:14.711135+0000","last_up_change":"2024-09-14T06:51:14.927617+0000","last_in_change":"2024-09-14T06:51:05.096335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T06:51:15.054238+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T06:51:19.403913+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9c9e29ab-ca41-4253-b78f-fd771e0b1ac9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6817","nonce":3974496147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6819","nonce":3974496147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6823","nonce":3974496147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3974496147},{"type":"v1","addr":"172.21.15.129:6821","nonce":3974496147}]},"public_addr":"172.21.15.129:6817/3974496147","cluster_addr":"172.21.15.129:6819/3974496147","heartbeat_back_addr":"172.21.15.129:6823/3974496147","heartbeat_front_addr":"172.21.15.129:6821/3974496147","state":["exists","up"]},{"osd":1,"uuid":"e2b7870c-f0b3-48f7-8c61-d69b195c1ad2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6801","nonce":498324347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6803","nonce":498324347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6807","nonce":498324347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":498324347},{"type":"v1","addr":"172.21.15.129:6805","nonce":498324347}]},"public_addr":"172.21.15.129:6801/498324347","cluster_addr":"172.21.15.129:6803/498324347","heartbeat_back_addr":"172.21.15.129:6807/498324347","heartbeat_front_addr":"172.21.15.129:6805/498324347","state":["exists","up"]},{"osd":2,"uuid":"8a45500a-5cc2-4dae-8608-00acc1982432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6809","nonce":3388405936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6811","nonce":3388405936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6815","nonce":3388405936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":3388405936},{"type":"v1","addr":"172.21.15.129:6813","nonce":3388405936}]},"public_addr":"172.21.15.129:6809/3388405936","cluster_addr":"172.21.15.129:6811/3388405936","heartbeat_back_addr":"172.21.15.129:6815/3388405936","heartbeat_front_addr":"172.21.15.129:6813/3388405936","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.879825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.057602+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T06:51:11.575932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T07:37:03.664 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.mgr.x has been restored 2024-09-14T07:37:04.296 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2024-09-14T07:37:04.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-14T07:37:04.460 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T07:37:04.460 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T07:37:04.460 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T07:37:04.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2024-09-14T07:37:04.753 INFO:teuthology.orchestra.run.smithi129.stderr:instructed osd(s) 0 to deep-scrub 2024-09-14T07:37:04.771 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2024-09-14T07:37:04.772 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-14T07:37:04.916 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T07:37:04.917 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T07:37:04.917 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T07:37:04.932 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2024-09-14T07:37:05.227 INFO:teuthology.orchestra.run.smithi129.stderr:instructed osd(s) 1 to deep-scrub 2024-09-14T07:37:05.247 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2024-09-14T07:37:05.247 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-14T07:37:05.394 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T07:37:05.394 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-14T07:37:05.394 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T07:37:05.411 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2024-09-14T07:37:05.695 INFO:teuthology.orchestra.run.smithi129.stderr:instructed osd(s) 2 to deep-scrub 2024-09-14T07:37:05.714 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T07:37:05.997 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T07:37:05.998 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T07:37:06.015 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":1707,"stamp":"2024-09-14T07:37:05.977056+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3818704,"kb_used_data":9232,"kb_used_omap":438,"kb_used_meta":3808969,"kb_avail":279296816,"statfs":{"total":289910292480,"available":285999939584,"internally_reserved":0,"allocated":9453568,"data_stored":5540101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":448962,"internal_metadata":3900384830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003076"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.731806+0000","last_change":"2024-09-14T06:51:23.124440+0000","last_active":"2024-09-14T07:36:14.731806+0000","last_peered":"2024-09-14T07:36:14.731806+0000","last_clean":"2024-09-14T07:36:14.731806+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T07:36:14.731806+0000","last_undegraded":"2024-09-14T07:36:14.731806+0000","last_fullsized":"2024-09-14T07:36:14.731806+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:54.063248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736557+0000","last_change":"2024-09-14T06:51:23.124277+0000","last_active":"2024-09-14T07:36:14.736557+0000","last_peered":"2024-09-14T07:36:14.736557+0000","last_clean":"2024-09-14T07:36:14.736557+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T07:36:14.736557+0000","last_undegraded":"2024-09-14T07:36:14.736557+0000","last_fullsized":"2024-09-14T07:36:14.736557+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:36:43.389449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.729160+0000","last_change":"2024-09-14T06:51:23.123529+0000","last_active":"2024-09-14T07:36:14.729160+0000","last_peered":"2024-09-14T07:36:14.729160+0000","last_clean":"2024-09-14T07:36:14.729160+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T07:36:14.729160+0000","last_undegraded":"2024-09-14T07:36:14.729160+0000","last_fullsized":"2024-09-14T07:36:14.729160+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:18.274940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.736272+0000","last_change":"2024-09-14T06:51:23.123532+0000","last_active":"2024-09-14T07:36:14.736272+0000","last_peered":"2024-09-14T07:36:14.736272+0000","last_clean":"2024-09-14T07:36:14.736272+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T07:36:14.736272+0000","last_undegraded":"2024-09-14T07:36:14.736272+0000","last_fullsized":"2024-09-14T07:36:14.736272+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:54:21.958324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":681,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.453786+0000","last_change":"2024-09-14T06:51:23.118966+0000","last_active":"2024-09-14T07:36:17.453786+0000","last_peered":"2024-09-14T07:36:17.453786+0000","last_clean":"2024-09-14T07:36:17.453786+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T07:36:17.453786+0000","last_undegraded":"2024-09-14T07:36:17.453786+0000","last_fullsized":"2024-09-14T07:36:17.453786+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:50:59.628972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484984+0000","last_change":"2024-09-14T06:51:23.113503+0000","last_active":"2024-09-14T07:36:17.484984+0000","last_peered":"2024-09-14T07:36:17.484984+0000","last_clean":"2024-09-14T07:36:17.484984+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T07:36:17.484984+0000","last_undegraded":"2024-09-14T07:36:17.484984+0000","last_fullsized":"2024-09-14T07:36:17.484984+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:14:15.559943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:17.484900+0000","last_change":"2024-09-14T06:51:23.113565+0000","last_active":"2024-09-14T07:36:17.484900+0000","last_peered":"2024-09-14T07:36:17.484900+0000","last_clean":"2024-09-14T07:36:17.484900+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T07:36:17.484900+0000","last_undegraded":"2024-09-14T07:36:17.484900+0000","last_fullsized":"2024-09-14T07:36:17.484900+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:25:39.862806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.734420+0000","last_change":"2024-09-14T06:51:23.123208+0000","last_active":"2024-09-14T07:36:14.734420+0000","last_peered":"2024-09-14T07:36:14.734420+0000","last_clean":"2024-09-14T07:36:14.734420+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T07:36:14.734420+0000","last_undegraded":"2024-09-14T07:36:14.734420+0000","last_fullsized":"2024-09-14T07:36:14.734420+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:20.086433+0000","last_clean_scrub_stamp":"2024-09-14T06:51:20.086433+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T17:34:58.546304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'52","reported_seq":748,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:36:14.725176+0000","last_change":"2024-09-14T06:51:17.941930+0000","last_active":"2024-09-14T07:36:14.725176+0000","last_peered":"2024-09-14T07:36:14.725176+0000","last_clean":"2024-09-14T07:36:14.725176+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T07:36:14.725176+0000","last_undegraded":"2024-09-14T07:36:14.725176+0000","last_fullsized":"2024-09-14T07:36:14.725176+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T06:51:16.017137+0000","last_clean_scrub_stamp":"2024-09-14T06:51:16.017137+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:35:40.900521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738925,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":703124,"kb_used_data":2688,"kb_used_omap":149,"kb_used_meta":700266,"kb_avail":93668716,"statfs":{"total":96636764160,"available":95916765184,"internally_reserved":0,"allocated":2752512,"data_stored":1453109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":153404,"internal_metadata":717072580},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738922,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1405344,"kb_used_data":3272,"kb_used_omap":137,"kb_used_meta":1401910,"kb_avail":92966496,"statfs":{"total":96636764160,"available":95197691904,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141048,"internal_metadata":1435556104},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738922,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1710236,"kb_used_data":3272,"kb_used_omap":150,"kb_used_meta":1706793,"kb_avail":92661604,"statfs":{"total":96636764160,"available":94885482496,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":154510,"internal_metadata":1747756146},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.016 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.017 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-14T06:51:20.086433+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=20, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.017 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-14T06:51:16.017137+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=6, tm_min=51, tm_sec=16, tm_wday=5, tm_yday=258, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=14, tm_hour=7, tm_min=37, tm_sec=3, tm_wday=5, tm_yday=258, tm_isdst=0) 2024-09-14T07:37:06.017 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-14T07:37:26.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-14T07:37:26.330 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T07:37:26.330 INFO:teuthology.orchestra.run.smithi129.stderr:dumped all 2024-09-14T07:37:26.349 INFO:teuthology.orchestra.run.smithi129.stdout:{"pg_ready":true,"pg_map":{"version":1717,"stamp":"2024-09-14T07:37:25.982156+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59,"ondisk_log_size":59,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3818688,"kb_used_data":9232,"kb_used_omap":438,"kb_used_meta":3808969,"kb_avail":279296832,"statfs":{"total":289910292480,"available":285999955968,"internally_reserved":0,"allocated":9453568,"data_stored":5540101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":448962,"internal_metadata":3900384830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003062"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":687,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:09.816522+0000","last_change":"2024-09-14T07:37:09.816522+0000","last_active":"2024-09-14T07:37:09.816522+0000","last_peered":"2024-09-14T07:37:09.816522+0000","last_clean":"2024-09-14T07:37:09.816522+0000","last_became_active":"2024-09-14T06:51:21.118295+0000","last_became_peered":"2024-09-14T06:51:21.118295+0000","last_unstale":"2024-09-14T07:37:09.816522+0000","last_undegraded":"2024-09-14T07:37:09.816522+0000","last_fullsized":"2024-09-14T07:37:09.816522+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:37:09.816280+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:37:09.816280+0000","last_clean_scrub_stamp":"2024-09-14T07:37:09.816280+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:49.654564+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0023382149999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":687,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:10.771999+0000","last_change":"2024-09-14T07:37:10.771999+0000","last_active":"2024-09-14T07:37:10.771999+0000","last_peered":"2024-09-14T07:37:10.771999+0000","last_clean":"2024-09-14T07:37:10.771999+0000","last_became_active":"2024-09-14T06:51:21.115559+0000","last_became_peered":"2024-09-14T06:51:21.115559+0000","last_unstale":"2024-09-14T07:37:10.771999+0000","last_undegraded":"2024-09-14T07:37:10.771999+0000","last_fullsized":"2024-09-14T07:37:10.771999+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:37:10.771810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:37:10.771810+0000","last_clean_scrub_stamp":"2024-09-14T07:37:10.771810+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:16.077770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002212173,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":687,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:06.733126+0000","last_change":"2024-09-14T07:37:06.733126+0000","last_active":"2024-09-14T07:37:06.733126+0000","last_peered":"2024-09-14T07:37:06.733126+0000","last_clean":"2024-09-14T07:37:06.733126+0000","last_became_active":"2024-09-14T06:51:21.115465+0000","last_became_peered":"2024-09-14T06:51:21.115465+0000","last_unstale":"2024-09-14T07:37:06.733126+0000","last_undegraded":"2024-09-14T07:37:06.733126+0000","last_fullsized":"2024-09-14T07:37:06.733126+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:37:06.732922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:37:06.732922+0000","last_clean_scrub_stamp":"2024-09-14T07:37:06.732922+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:19:53.426450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001713502,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":688,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:07.731882+0000","last_change":"2024-09-14T07:37:07.731751+0000","last_active":"2024-09-14T07:37:07.731882+0000","last_peered":"2024-09-14T07:37:07.731882+0000","last_clean":"2024-09-14T07:37:07.731882+0000","last_became_active":"2024-09-14T06:51:21.116157+0000","last_became_peered":"2024-09-14T06:51:21.116157+0000","last_unstale":"2024-09-14T07:37:07.731882+0000","last_undegraded":"2024-09-14T07:37:07.731882+0000","last_fullsized":"2024-09-14T07:37:07.731882+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:37:07.731568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:37:07.731568+0000","last_clean_scrub_stamp":"2024-09-14T07:37:07.731568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:13:23.912287+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.001667473,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"334'3","reported_seq":691,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:05.405191+0000","last_change":"2024-09-14T07:37:05.405114+0000","last_active":"2024-09-14T07:37:05.405191+0000","last_peered":"2024-09-14T07:37:05.405191+0000","last_clean":"2024-09-14T07:37:05.405191+0000","last_became_active":"2024-09-14T06:51:21.123157+0000","last_became_peered":"2024-09-14T06:51:21.123157+0000","last_unstale":"2024-09-14T07:37:05.405191+0000","last_undegraded":"2024-09-14T07:37:05.405191+0000","last_fullsized":"2024-09-14T07:37:05.405191+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"334'3","last_scrub_stamp":"2024-09-14T07:37:05.405040+0000","last_deep_scrub":"334'3","last_deep_scrub_stamp":"2024-09-14T07:37:05.405040+0000","last_clean_scrub_stamp":"2024-09-14T07:37:05.405040+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:14:40.052540+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.001861066,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":686,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:06.343891+0000","last_change":"2024-09-14T07:37:06.343708+0000","last_active":"2024-09-14T07:37:06.343891+0000","last_peered":"2024-09-14T07:37:06.343891+0000","last_clean":"2024-09-14T07:37:06.343891+0000","last_became_active":"2024-09-14T06:51:21.113688+0000","last_became_peered":"2024-09-14T06:51:21.113688+0000","last_unstale":"2024-09-14T07:37:06.343891+0000","last_undegraded":"2024-09-14T07:37:06.343891+0000","last_fullsized":"2024-09-14T07:37:06.343891+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:37:06.343527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:37:06.343527+0000","last_clean_scrub_stamp":"2024-09-14T07:37:06.343527+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:51:03.049136+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00084500000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":686,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:07.365885+0000","last_change":"2024-09-14T07:37:07.365885+0000","last_active":"2024-09-14T07:37:07.365885+0000","last_peered":"2024-09-14T07:37:07.365885+0000","last_clean":"2024-09-14T07:37:07.365885+0000","last_became_active":"2024-09-14T06:51:21.113286+0000","last_became_peered":"2024-09-14T06:51:21.113286+0000","last_unstale":"2024-09-14T07:37:07.365885+0000","last_undegraded":"2024-09-14T07:37:07.365885+0000","last_fullsized":"2024-09-14T07:37:07.365885+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T07:37:07.365676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T07:37:07.365676+0000","last_clean_scrub_stamp":"2024-09-14T07:37:07.365676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:19:35.500840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092666299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"334'2","reported_seq":690,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:08.777866+0000","last_change":"2024-09-14T07:37:08.777723+0000","last_active":"2024-09-14T07:37:08.777866+0000","last_peered":"2024-09-14T07:37:08.777866+0000","last_clean":"2024-09-14T07:37:08.777866+0000","last_became_active":"2024-09-14T06:51:21.113159+0000","last_became_peered":"2024-09-14T06:51:21.113159+0000","last_unstale":"2024-09-14T07:37:08.777866+0000","last_undegraded":"2024-09-14T07:37:08.777866+0000","last_fullsized":"2024-09-14T07:37:08.777866+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"334'2","last_scrub_stamp":"2024-09-14T07:37:08.777529+0000","last_deep_scrub":"334'2","last_deep_scrub_stamp":"2024-09-14T07:37:08.777529+0000","last_clean_scrub_stamp":"2024-09-14T07:37:08.777529+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T18:39:59.456700+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00146842,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"334'54","reported_seq":758,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-14T07:37:05.700237+0000","last_change":"2024-09-14T07:37:05.700237+0000","last_active":"2024-09-14T07:37:05.700237+0000","last_peered":"2024-09-14T07:37:05.700237+0000","last_clean":"2024-09-14T07:37:05.700237+0000","last_became_active":"2024-09-14T06:51:17.939323+0000","last_became_peered":"2024-09-14T06:51:17.939323+0000","last_unstale":"2024-09-14T07:37:05.700237+0000","last_undegraded":"2024-09-14T07:37:05.700237+0000","last_fullsized":"2024-09-14T07:37:05.700237+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"334'54","last_scrub_stamp":"2024-09-14T07:37:05.700164+0000","last_deep_scrub":"334'54","last_deep_scrub_stamp":"2024-09-14T07:37:05.700164+0000","last_clean_scrub_stamp":"2024-09-14T07:37:05.700164+0000","objects_scrubbed":2,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:40:31.394874+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54,"ondisk_log_size":54,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738929,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":703120,"kb_used_data":2688,"kb_used_omap":149,"kb_used_meta":700266,"kb_avail":93668720,"statfs":{"total":96636764160,"available":95916769280,"internally_reserved":0,"allocated":2752512,"data_stored":1453109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":153404,"internal_metadata":717072580},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738926,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1405336,"kb_used_data":3272,"kb_used_omap":137,"kb_used_meta":1401910,"kb_avail":92966504,"statfs":{"total":96636764160,"available":95197700096,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":141048,"internal_metadata":1435556104},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738926,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1710232,"kb_used_data":3272,"kb_used_omap":150,"kb_used_meta":1706793,"kb_avail":92661608,"statfs":{"total":96636764160,"available":94885486592,"internally_reserved":0,"allocated":3350528,"data_stored":2043496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":154510,"internal_metadata":1747756146},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T07:37:26.350 DEBUG:teuthology.orchestra.run.smithi129:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2024-09-14T07:37:26.673 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-14T07:37:26.673 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-14T07:37:26.673 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2024-09-14T07:37:26.674 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:37:32.381 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 is failed for ~0s 2024-09-14T07:37:32.775 INFO:tasks.ceph.cluster1.osd.0:Stopped 2024-09-14T07:37:32.776 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2024-09-14T07:37:32.776 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:37:37.684 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 is failed for ~0s 2024-09-14T07:37:37.685 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 has been restored 2024-09-14T07:37:38.878 INFO:tasks.ceph.cluster1.osd.1:Stopped 2024-09-14T07:37:38.878 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2024-09-14T07:37:38.879 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:37:38.970 INFO:tasks.ceph.cluster1.osd.2:Stopped 2024-09-14T07:37:38.970 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-14T07:37:38.970 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2024-09-14T07:37:38.970 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:37:38.998 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2024-09-14T07:37:38.998 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-14T07:37:38.998 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2024-09-14T07:37:38.998 INFO:teuthology.orchestra.run:waiting for 300 2024-09-14T07:37:39.055 INFO:tasks.ceph.cluster1.mon.a:Stopped 2024-09-14T07:37:39.055 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-14T07:37:39.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-14T07:37:39.150 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T07:37:39.150 DEBUG:teuthology.orchestra.run.smithi129:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2024-09-14T07:37:39.553 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T07:37:39.553 DEBUG:teuthology.orchestra.run.smithi129:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2024-09-14T07:37:39.860 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T07:37:39.860 DEBUG:teuthology.orchestra.run.smithi129:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2024-09-14T07:37:40.184 INFO:tasks.ceph:Archiving mon data... 2024-09-14T07:37:40.184 DEBUG:teuthology.misc:Transferring archived files from smithi129:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/data/cluster1.mon.a.tgz 2024-09-14T07:37:40.184 DEBUG:teuthology.orchestra.run.smithi129:> mktemp 2024-09-14T07:37:40.202 INFO:teuthology.orchestra.run.smithi129.stdout:/tmp/tmp.wpV4hslgjI 2024-09-14T07:37:40.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.wpV4hslgjI 2024-09-14T07:37:40.368 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0666 /tmp/tmp.wpV4hslgjI 2024-09-14T07:37:40.464 DEBUG:teuthology.orchestra.remote:smithi129:/tmp/tmp.wpV4hslgjI is 394KB 2024-09-14T07:37:40.526 DEBUG:teuthology.orchestra.run.smithi129:> rm -fr /tmp/tmp.wpV4hslgjI 2024-09-14T07:37:40.530 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-14T07:37:40.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T07:37:40.578 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-14T07:37:40.648 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-14T07:37:40.664 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-14T07:37:40.664 INFO:tasks.ceph:Archiving crash dumps... 2024-09-14T07:37:40.666 DEBUG:teuthology.misc:Transferring archived files from smithi129:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/remote/smithi129/crash 2024-09-14T07:37:40.667 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-14T07:37:40.704 DEBUG:teuthology.misc:Transferring archived files from smithi169:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/remote/smithi169/crash 2024-09-14T07:37:40.705 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-14T07:37:40.720 INFO:tasks.ceph:Compressing logs... 2024-09-14T07:37:40.720 DEBUG:teuthology.orchestra.run.smithi129:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T07:37:40.750 DEBUG:teuthology.orchestra.run.smithi169:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T07:37:40.764 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31393.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25282.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31393.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30008.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31393.log.gz 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.25282.log: gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25282.log.gz 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.30008.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29548.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30008.log.gz 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31523.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29548.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29548.log.gz 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.21578.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29670.log 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31523.log.gz 2024-09-14T07:37:40.767 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/ceph.tmp-client.admin.21578.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29808.log 2024-09-14T07:37:40.768 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.21578.log.gz 2024-09-14T07:37:40.768 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29670.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29670.log.gz 2024-09-14T07:37:40.768 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2024-09-14T07:37:40.779 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29644.log 2024-09-14T07:37:40.779 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29808.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29808.log.gz 2024-09-14T07:37:40.779 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28749.log 2024-09-14T07:37:40.779 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29644.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31471.log 2024-09-14T07:37:40.779 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29644.log.gz 2024-09-14T07:37:40.779 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31627.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.28749.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28749.log.gz 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29956.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31471.log.gz 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31627.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31627.log.gz 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31653.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29956.log.gz 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29598.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31445.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31653.log.gz 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29696.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31445.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31445.log.gz 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2024-09-14T07:37:40.780 INFO:teuthology.orchestra.run.smithi129.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.29598.log.gz 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29696.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31367.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29696.log.gz 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31549.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31367.log.gz 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29886.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31549.log.gz 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25188.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29886.log.gz 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25306.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.25188.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25365.log 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25188.log.gz 2024-09-14T07:37:40.781 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.25306.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25306.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31575.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.admin.25365.log: /var/log/ceph/cluster1-client.admin.31601.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25365.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25330.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31575.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31601.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31601.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29982.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.25330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31497.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25330.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29982.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29573.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29982.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29882.log 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31497.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29573.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29573.log.gz 2024-09-14T07:37:40.782 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29726.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29730.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29882.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29726.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29726.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31677.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29730.log: gzip -5 --verbose 0.0% -- -- replaced with /var/log/ceph/cluster1-client.admin.29730.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr: /var/log/ceph/cluster1-client.admin.29878.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.31677.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29618.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31677.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29878.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster1.log -- replaced with /var/log/ceph/cluster1-client.admin.29878.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr: 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29729.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29618.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25431.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29807.log 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29729.log.gz 2024-09-14T07:37:40.783 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.25431.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25431.log.gz 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29800.log 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-client.admin.29807.log: /var/log/ceph/cluster1-client.admin.29800.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29807.log.gz 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29800.log.gz 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31419.log 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/cluster1-osd.1.log: /var/log/ceph/cluster1-client.admin.31419.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31419.log.gz 2024-09-14T07:37:40.784 INFO:teuthology.orchestra.run.smithi129.stderr: 88.0% -- replaced with /var/log/ceph/cluster1.log.gz 2024-09-14T07:37:40.785 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105419.log 2024-09-14T07:37:40.785 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124764.log 2024-09-14T07:37:40.785 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85872.log 2024-09-14T07:37:40.786 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105419.log: /var/log/ceph/cluster2-client.mirror.124764.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128730.log 2024-09-14T07:37:40.786 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85872.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105419.log.gz 2024-09-14T07:37:40.786 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108673.log 2024-09-14T07:37:40.786 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128730.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124764.log.gz 2024-09-14T07:37:40.786 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56254.log 2024-09-14T07:37:40.786 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108673.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128730.log.gz 2024-09-14T07:37:40.788 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112070.log 2024-09-14T07:37:40.788 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108673.log.gz 2024-09-14T07:37:40.788 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101008.log 2024-09-14T07:37:40.788 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46769.log 2024-09-14T07:37:40.788 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.101008.log: -- /var/log/ceph/cluster2-client.mirror.132369.log 2024-09-14T07:37:40.788 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101481.log 2024-09-14T07:37:40.789 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59195.log 2024-09-14T07:37:40.789 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88422.log 2024-09-14T07:37:40.789 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101481.log.gz 2024-09-14T07:37:40.789 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101008.log.gz 2024-09-14T07:37:40.789 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59195.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122539.log 2024-09-14T07:37:40.790 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34395.log 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88422.log: /var/log/ceph/cluster2-client.mirror.122539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128865.log 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.132369.log.gz 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122539.log.gz 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45452.log 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85402.log 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128865.log.gz 2024-09-14T07:37:40.791 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45452.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78989.log 2024-09-14T07:37:40.792 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85402.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133467.log 2024-09-14T07:37:40.792 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78989.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130734.log 2024-09-14T07:37:40.792 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112409.log 2024-09-14T07:37:40.792 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133467.log: /var/log/ceph/cluster2-client.mirror.130734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47455.log 2024-09-14T07:37:40.792 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.133467.log.gz 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112409.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30530.log 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112409.log.gz 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112896.log 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30530.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110914.log 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.130734.log.gz 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.112896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133489.log 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43619.log 2024-09-14T07:37:40.795 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.112896.log.gz 2024-09-14T07:37:40.796 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133489.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.110914.log.gz 2024-09-14T07:37:40.796 INFO:teuthology.orchestra.run.smithi169.stderr: 73.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30530.log.gz 2024-09-14T07:37:40.796 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95323.log 2024-09-14T07:37:40.796 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85872.log.gz 2024-09-14T07:37:40.796 INFO:teuthology.orchestra.run.smithi169.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133489.log.gz 2024-09-14T07:37:40.796 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77188.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124257.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95323.log.gz 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77188.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45529.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124257.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32345.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.45529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30155.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124257.log.gz 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95666.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46769.log.gz 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126349.log 2024-09-14T07:37:40.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30155.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30155.log.gz 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81417.log 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126349.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95666.log.gz 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77223.log 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126349.log.gz 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81417.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132073.log 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73860.log 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132073.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58114.log 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr: 85.5% -- replaced with /var/log/ceph/cluster2-client.mirror.132073.log.gz 2024-09-14T07:37:40.800 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56126.log 2024-09-14T07:37:40.803 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73860.log: /var/log/ceph/cluster2-client.mirror.58114.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124419.log 2024-09-14T07:37:40.803 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56254.log.gz 2024-09-14T07:37:40.803 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86049.log 2024-09-14T07:37:40.803 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124419.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75115.log 2024-09-14T07:37:40.803 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.86049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104315.log 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75115.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115506.log 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.104315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132431.log 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.115506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92946.log 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124419.log.gz 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98953.log 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.92946.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132431.log.gz 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93097.log 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.92946.log.gz 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98953.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98953.log.gz 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104315.log.gz 2024-09-14T07:37:40.804 INFO:teuthology.orchestra.run.smithi169.stderr: 90.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115506.log.gz 2024-09-14T07:37:40.807 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132838.log 2024-09-14T07:37:40.807 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104837.log 2024-09-14T07:37:40.807 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93097.log.gz 2024-09-14T07:37:40.807 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94069.log 2024-09-14T07:37:40.807 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132838.log.gz/var/log/ceph/cluster2-client.mirror.104837.log: 2024-09-14T07:37:40.807 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44363.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.94069.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104837.log.gz 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126173.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94069.log.gz 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129090.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126173.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131124.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130336.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129090.log.gz 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126173.log.gz 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131124.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72467.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130336.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131124.log.gz 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.134082.log 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130336.log.gz 2024-09-14T07:37:40.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73395.log 2024-09-14T07:37:40.810 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.134082.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129158.log 2024-09-14T07:37:40.810 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.134082.log.gz 2024-09-14T07:37:40.810 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116183.log 2024-09-14T07:37:40.810 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36721.log 2024-09-14T07:37:40.810 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129158.log.gz 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122616.log 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95913.log 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107212.log 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122616.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81417.log.gz 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95913.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122616.log.gz 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56126.log.gz 2024-09-14T07:37:40.811 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95913.log.gz 2024-09-14T07:37:40.812 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88164.log 2024-09-14T07:37:40.812 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.107212.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96003.log 2024-09-14T07:37:40.813 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124152.log 2024-09-14T07:37:40.813 INFO:teuthology.orchestra.run.smithi169.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88164.log.gz 2024-09-14T07:37:40.813 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132627.log 2024-09-14T07:37:40.814 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119785.log 2024-09-14T07:37:40.814 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.96003.log.gz 2024-09-14T07:37:40.814 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124152.log: /var/log/ceph/cluster1-client.mirror.132627.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.107212.log.gz 2024-09-14T07:37:40.814 INFO:teuthology.orchestra.run.smithi169.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.132627.log.gz 2024-09-14T07:37:40.815 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124152.log.gz 2024-09-14T07:37:40.815 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64169.log 2024-09-14T07:37:40.816 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32345.log.gz 2024-09-14T07:37:40.816 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64727.log 2024-09-14T07:37:40.816 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64169.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119785.log.gz 2024-09-14T07:37:40.816 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36686.log 2024-09-14T07:37:40.816 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126866.log 2024-09-14T07:37:40.817 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36686.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131942.log 2024-09-14T07:37:40.818 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.30430.log 2024-09-14T07:37:40.818 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81518.log 2024-09-14T07:37:40.818 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.131942.log.gz 2024-09-14T07:37:40.818 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.30430.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121611.log 2024-09-14T07:37:40.818 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34395.log.gz 2024-09-14T07:37:40.818 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34860.log 2024-09-14T07:37:40.819 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.121611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.69374.log 2024-09-14T07:37:40.819 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126866.log.gz 2024-09-14T07:37:40.820 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88093.log 2024-09-14T07:37:40.820 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34860.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.121611.log.gz 2024-09-14T07:37:40.820 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.69374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64320.log 2024-09-14T07:37:40.820 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30280.log 2024-09-14T07:37:40.820 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88093.log.gz 2024-09-14T07:37:40.820 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64320.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109830.log 2024-09-14T07:37:40.821 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.122082.log 2024-09-14T07:37:40.821 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30280.log.gz 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25312.log 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72467.log.gz 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109830.log.gz 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128753.log 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.2.122082.log: /var/log/ceph/cluster2-client.admin.25312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109447.log 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25312.log.gz 2024-09-14T07:37:40.822 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85599.log 2024-09-14T07:37:40.823 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128753.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99166.log 2024-09-14T07:37:40.823 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.109447.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.128753.log.gz 2024-09-14T07:37:40.823 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85599.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36686.log.gz 2024-09-14T07:37:40.823 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36721.log.gz 2024-09-14T07:37:40.823 INFO:teuthology.orchestra.run.smithi169.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109447.log.gz 2024-09-14T07:37:40.825 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86049.log.gz 2024-09-14T07:37:40.826 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45452.log.gz 2024-09-14T07:37:40.828 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44770.log 2024-09-14T07:37:40.828 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44677.log 2024-09-14T07:37:40.829 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.44770.log: -- replaced with /var/log/ceph/cluster2-client.mirror.99166.log.gz 2024-09-14T07:37:40.829 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59195.log.gz 2024-09-14T07:37:40.830 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80776.log 2024-09-14T07:37:40.830 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132390.log 2024-09-14T07:37:40.830 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106114.log 2024-09-14T07:37:40.832 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83336.log 2024-09-14T07:37:40.832 INFO:teuthology.orchestra.run.smithi169.stderr: 83.6%/var/log/ceph/cluster2-client.mirror.106114.log: -- replaced with /var/log/ceph/cluster2-client.mirror.132390.log.gz 2024-09-14T07:37:40.832 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65457.log 2024-09-14T07:37:40.832 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106114.log.gz 2024-09-14T07:37:40.832 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78787.log 2024-09-14T07:37:40.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119526.log 2024-09-14T07:37:40.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89395.log 2024-09-14T07:37:40.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87402.log 2024-09-14T07:37:40.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89395.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119526.log.gz 2024-09-14T07:37:40.833 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58858.log 2024-09-14T07:37:40.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87402.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85599.log.gz 2024-09-14T07:37:40.836 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118864.log 2024-09-14T07:37:40.836 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97163.log 2024-09-14T07:37:40.836 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127853.log 2024-09-14T07:37:40.837 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118864.log: /var/log/ceph/cluster2-client.mirror.97163.log: 88.2% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64727.log.gz 2024-09-14T07:37:40.838 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.118864.log.gz 2024-09-14T07:37:40.838 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97163.log.gz 2024-09-14T07:37:40.838 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125611.log 2024-09-14T07:37:40.839 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106809.log 2024-09-14T07:37:40.839 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127853.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127853.log.gz/var/log/ceph/cluster2-client.mirror.125611.log: 2024-09-14T07:37:40.839 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84311.log 2024-09-14T07:37:40.839 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125611.log.gz 2024-09-14T07:37:40.839 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106809.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.34860.log.gz 2024-09-14T07:37:40.839 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.86862.log 2024-09-14T07:37:40.840 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106809.log.gz 2024-09-14T07:37:40.840 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84311.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127178.log 2024-09-14T07:37:40.841 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126824.log 2024-09-14T07:37:40.841 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.86862.log: /var/log/ceph/cluster1-client.mirror.127178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69898.log 2024-09-14T07:37:40.841 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106219.log 2024-09-14T07:37:40.841 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127178.log.gz 2024-09-14T07:37:40.841 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69898.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126824.log.gz 2024-09-14T07:37:40.842 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106219.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73395.log.gz 2024-09-14T07:37:40.842 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73988.log 2024-09-14T07:37:40.842 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84753.log 2024-09-14T07:37:40.843 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90953.log 2024-09-14T07:37:40.843 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73860.log.gz 2024-09-14T07:37:40.844 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130377.log 2024-09-14T07:37:40.844 INFO:teuthology.orchestra.run.smithi169.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106219.log.gz/var/log/ceph/cluster2-client.mirror.90953.log: 2024-09-14T07:37:40.844 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84753.log: 84.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119155.log 2024-09-14T07:37:40.845 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.90953.log.gz 2024-09-14T07:37:40.845 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130377.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130377.log.gz 2024-09-14T07:37:40.845 INFO:teuthology.orchestra.run.smithi129.stderr: 93.0% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125141.log 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119155.log: 92.4% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69898.log.gz 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.89395.log.gz 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119155.log.gz 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65784.log 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131303.log 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65784.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97526.log 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125141.log.gz 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39518.log 2024-09-14T07:37:40.847 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97526.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129338.log 2024-09-14T07:37:40.848 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57614.log 2024-09-14T07:37:40.848 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64169.log.gz 2024-09-14T07:37:40.848 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129338.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97526.log.gz 2024-09-14T07:37:40.849 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129338.log.gz 2024-09-14T07:37:40.849 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64320.log.gz 2024-09-14T07:37:40.849 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39518.log.gz 2024-09-14T07:37:40.851 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73988.log.gz 2024-09-14T07:37:40.852 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51051.log 2024-09-14T07:37:40.852 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50121.log 2024-09-14T07:37:40.853 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91091.log 2024-09-14T07:37:40.853 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51051.log: /var/log/ceph/cluster2-client.mirror.50121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81584.log 2024-09-14T07:37:40.853 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123338.log 2024-09-14T07:37:40.853 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91091.log: /var/log/ceph/cluster2-client.mirror.81584.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91091.log.gz 2024-09-14T07:37:40.854 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60968.log 2024-09-14T07:37:40.854 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123338.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45702.log 2024-09-14T07:37:40.855 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43154.log 2024-09-14T07:37:40.856 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45702.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29930.log 2024-09-14T07:37:40.856 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.123338.log.gz 2024-09-14T07:37:40.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51516.log 2024-09-14T07:37:40.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85851.log 2024-09-14T07:37:40.857 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29930.log.gz 2024-09-14T07:37:40.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51516.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78651.log 2024-09-14T07:37:40.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85851.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78787.log.gz 2024-09-14T07:37:40.858 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103365.log 2024-09-14T07:37:40.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78651.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128708.log 2024-09-14T07:37:40.859 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78989.log.gz 2024-09-14T07:37:40.859 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122398.log 2024-09-14T07:37:40.859 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103365.log: /var/log/ceph/cluster1-client.mirror.128708.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103365.log.gz 2024-09-14T07:37:40.859 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.128708.log.gz 2024-09-14T07:37:40.862 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.122082.log.gz 2024-09-14T07:37:40.862 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57742.log 2024-09-14T07:37:40.863 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122398.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131504.log 2024-09-14T07:37:40.863 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122398.log.gz 2024-09-14T07:37:40.863 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116738.log 2024-09-14T07:37:40.863 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85359.log 2024-09-14T07:37:40.864 INFO:teuthology.orchestra.run.smithi169.stderr:gzip 88.4% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.131504.log.gz 2024-09-14T07:37:40.864 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.77688.log 2024-09-14T07:37:40.864 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116738.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116738.log.gz 2024-09-14T07:37:40.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48850.log 2024-09-14T07:37:40.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77281.log 2024-09-14T07:37:40.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112028.log 2024-09-14T07:37:40.866 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77281.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91840.log 2024-09-14T07:37:40.866 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126491.log 2024-09-14T07:37:40.866 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131795.log 2024-09-14T07:37:40.866 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126491.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126491.log.gz 2024-09-14T07:37:40.867 INFO:teuthology.orchestra.run.smithi169.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112028.log.gz 2024-09-14T07:37:40.868 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114418.log 2024-09-14T07:37:40.868 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52260.log 2024-09-14T07:37:40.869 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.114418.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131795.log.gz 2024-09-14T07:37:40.870 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.114418.log.gz 2024-09-14T07:37:40.873 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80776.log.gz 2024-09-14T07:37:40.873 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126988.log 2024-09-14T07:37:40.873 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99131.log 2024-09-14T07:37:40.874 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74139.log 2024-09-14T07:37:40.874 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99131.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126988.log.gz 2024-09-14T07:37:40.874 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99131.log.gz 2024-09-14T07:37:40.875 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34639.log 2024-09-14T07:37:40.875 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74139.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95435.log 2024-09-14T07:37:40.876 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66848.log 2024-09-14T07:37:40.876 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95467.log 2024-09-14T07:37:40.876 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95435.log: /var/log/ceph/cluster2-client.mirror.66848.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95435.log.gz 2024-09-14T07:37:40.877 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86120.log 2024-09-14T07:37:40.877 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.95467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36500.log 2024-09-14T07:37:40.877 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95467.log.gz 2024-09-14T07:37:40.878 INFO:teuthology.orchestra.run.smithi169.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85851.log.gz 2024-09-14T07:37:40.878 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32252.log 2024-09-14T07:37:40.878 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36500.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122929.log 2024-09-14T07:37:40.879 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74081.log 2024-09-14T07:37:40.879 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122929.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127059.log 2024-09-14T07:37:40.879 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74081.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57614.log.gz 2024-09-14T07:37:40.881 INFO:teuthology.orchestra.run.smithi169.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.122929.log.gz 2024-09-14T07:37:40.881 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58114.log.gz 2024-09-14T07:37:40.882 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32252.log.gz 2024-09-14T07:37:40.882 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114805.log 2024-09-14T07:37:40.882 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.127059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129248.log 2024-09-14T07:37:40.882 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112640.log 2024-09-14T07:37:40.882 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.127059.log.gz 2024-09-14T07:37:40.883 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114805.log: /var/log/ceph/cluster1-client.mirror.129248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131403.log 2024-09-14T07:37:40.883 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129248.log.gz 2024-09-14T07:37:40.883 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112640.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.112640.log.gz 2024-09-14T07:37:40.883 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122504.log 2024-09-14T07:37:40.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131403.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108792.log 2024-09-14T07:37:40.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122504.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122504.log.gz 2024-09-14T07:37:40.886 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.114805.log.gz 2024-09-14T07:37:40.888 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32531.log 2024-09-14T07:37:40.888 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108792.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83806.log 2024-09-14T07:37:40.889 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108792.log.gz 2024-09-14T07:37:40.889 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83806.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57742.log.gz 2024-09-14T07:37:40.890 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35140.log 2024-09-14T07:37:40.893 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32531.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30000.log 2024-09-14T07:37:40.893 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76572.log 2024-09-14T07:37:40.893 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71816.log 2024-09-14T07:37:40.893 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30000.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30000.log.gz 2024-09-14T07:37:40.894 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125806.log 2024-09-14T07:37:40.894 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122082.log 2024-09-14T07:37:40.895 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125806.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125806.log.gz 2024-09-14T07:37:40.895 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115073.log 2024-09-14T07:37:40.895 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47455.log.gz 2024-09-14T07:37:40.896 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88469.log 2024-09-14T07:37:40.897 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115073.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115073.log.gz 2024-09-14T07:37:40.897 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52260.log.gz 2024-09-14T07:37:40.900 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.112070.log.gz 2024-09-14T07:37:40.900 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76758.log 2024-09-14T07:37:40.901 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88469.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88469.log.gz 2024-09-14T07:37:40.901 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34639.log.gz 2024-09-14T07:37:40.902 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60968.log.gz 2024-09-14T07:37:40.904 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100789.log 2024-09-14T07:37:40.904 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37279.log 2024-09-14T07:37:40.905 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76758.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128168.log 2024-09-14T07:37:40.905 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100789.log: /var/log/ceph/cluster2-client.mirror.37279.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83336.log.gz 2024-09-14T07:37:40.906 INFO:teuthology.orchestra.run.smithi169.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76630.log 2024-09-14T07:37:40.906 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.100789.log.gz 2024-09-14T07:37:40.906 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128168.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30455.log 2024-09-14T07:37:40.906 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128168.log.gz 2024-09-14T07:37:40.907 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78752.log 2024-09-14T07:37:40.907 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76630.log: /var/log/ceph/cluster2-client.mirror.30455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.123505.log 2024-09-14T07:37:40.907 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60014.log 2024-09-14T07:37:40.907 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44119.log 2024-09-14T07:37:40.908 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.2.123505.log: /var/log/ceph/cluster2-client.mirror.60014.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66848.log.gz 2024-09-14T07:37:40.908 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67557.log 2024-09-14T07:37:40.909 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104980.log 2024-09-14T07:37:40.910 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123432.log 2024-09-14T07:37:40.910 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74139.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.74081.log.gz 2024-09-14T07:37:40.910 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:40.911 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104980.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104980.log.gz 2024-09-14T07:37:40.911 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66005.log 2024-09-14T07:37:40.913 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123432.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.123432.log.gz 2024-09-14T07:37:40.913 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46246.log 2024-09-14T07:37:40.914 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43619.log.gz 2024-09-14T07:37:40.914 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126917.log 2024-09-14T07:37:40.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46246.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44363.log.gz 2024-09-14T07:37:40.915 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44677.log.gz 2024-09-14T07:37:40.915 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44770.log.gz 2024-09-14T07:37:40.916 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131255.log 2024-09-14T07:37:40.916 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43154.log.gz 2024-09-14T07:37:40.918 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126917.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32531.log.gz 2024-09-14T07:37:40.918 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85402.log.gz 2024-09-14T07:37:40.918 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126917.log.gz 2024-09-14T07:37:40.919 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42538.log 2024-09-14T07:37:40.919 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.2.123505.log.gz 2024-09-14T07:37:40.919 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71816.log.gz 2024-09-14T07:37:40.919 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131255.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.131255.log.gz 2024-09-14T07:37:40.919 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46246.log.gz 2024-09-14T07:37:40.928 INFO:teuthology.orchestra.run.smithi169.stderr: 81.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30455.log.gz 2024-09-14T07:37:40.928 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58858.log.gz 2024-09-14T07:37:40.929 INFO:teuthology.orchestra.run.smithi169.stderr: 81.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60014.log.gz 2024-09-14T07:37:40.929 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44119.log.gz 2024-09-14T07:37:40.931 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83806.log.gz 2024-09-14T07:37:40.932 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36535.log 2024-09-14T07:37:40.933 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97086.log 2024-09-14T07:37:40.933 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42538.log: /var/log/ceph/cluster2-client.mirror.36535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.84415.log 2024-09-14T07:37:40.934 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109200.log 2024-09-14T07:37:40.934 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97086.log.gz 2024-09-14T07:37:40.934 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.2.84415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29506.log 2024-09-14T07:37:40.934 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48443.log 2024-09-14T07:37:40.935 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%/var/log/ceph/cluster2-client.admin.29506.log: -- replaced with /var/log/ceph/cluster2-client.mirror.109200.log.gz 2024-09-14T07:37:40.935 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29506.log.gz 2024-09-14T07:37:40.935 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126653.log 2024-09-14T07:37:40.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69154.log 2024-09-14T07:37:40.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75417.log 2024-09-14T07:37:40.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123225.log 2024-09-14T07:37:40.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75417.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126653.log.gz 2024-09-14T07:37:40.938 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46804.log 2024-09-14T07:37:40.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123225.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.123225.log.gz 2024-09-14T07:37:40.940 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35140.log.gz 2024-09-14T07:37:40.940 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49935.log 2024-09-14T07:37:40.941 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52167.log 2024-09-14T07:37:40.943 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46804.log: /var/log/ceph/cluster2-client.mirror.49935.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67557.log.gz 2024-09-14T07:37:40.944 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80978.log 2024-09-14T07:37:40.944 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126053.log 2024-09-14T07:37:40.945 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30879.log 2024-09-14T07:37:40.945 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103583.log 2024-09-14T07:37:40.946 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126053.log: /var/log/ceph/cluster1-client.mirror.30879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67092.log 2024-09-14T07:37:40.946 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126053.log.gz 2024-09-14T07:37:40.946 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103583.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85101.log 2024-09-14T07:37:40.947 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.67092.log: -- replaced with /var/log/ceph/cluster2-client.mirror.103583.log.gz 2024-09-14T07:37:40.947 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107677.log 2024-09-14T07:37:40.947 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62588.log 2024-09-14T07:37:40.948 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63890.log 2024-09-14T07:37:40.949 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107677.log: /var/log/ceph/cluster2-client.mirror.62588.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.107677.log.gz 2024-09-14T07:37:40.949 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68391.log 2024-09-14T07:37:40.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57556.log 2024-09-14T07:37:40.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90253.log 2024-09-14T07:37:40.951 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60503.log 2024-09-14T07:37:40.951 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57556.log: /var/log/ceph/cluster2-client.mirror.90253.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47269.log 2024-09-14T07:37:40.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60503.log: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131403.log.gz 2024-09-14T07:37:40.953 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35419.log 2024-09-14T07:37:40.954 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.47269.log: --verbose -- /var/log/ceph/cluster2-client.mirror.43654.log 2024-09-14T07:37:40.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35419.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55510.log 2024-09-14T07:37:40.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89267.log 2024-09-14T07:37:40.956 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59868.log 2024-09-14T07:37:40.956 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89267.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89267.log.gz 2024-09-14T07:37:40.957 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131483.log 2024-09-14T07:37:40.958 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59868.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76909.log 2024-09-14T07:37:40.958 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131483.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48443.log.gz 2024-09-14T07:37:40.958 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.30430.log.gz 2024-09-14T07:37:40.959 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.131483.log.gz 2024-09-14T07:37:40.959 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133181.log 2024-09-14T07:37:40.959 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33240.log 2024-09-14T07:37:40.960 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76909.log: /var/log/ceph/cluster2-client.mirror.133181.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47362.log 2024-09-14T07:37:40.960 INFO:teuthology.orchestra.run.smithi169.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.133181.log.gz 2024-09-14T07:37:40.960 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33240.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46583.log 2024-09-14T07:37:40.961 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84902.log 2024-09-14T07:37:40.961 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47362.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42538.log.gz 2024-09-14T07:37:40.963 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46583.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36500.log.gz 2024-09-14T07:37:40.964 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36535.log.gz 2024-09-14T07:37:40.964 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107719.log 2024-09-14T07:37:40.964 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66941.log 2024-09-14T07:37:40.965 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74453.log 2024-09-14T07:37:40.965 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107719.log: /var/log/ceph/cluster2-client.mirror.66941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80473.log 2024-09-14T07:37:40.966 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74453.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82023.log 2024-09-14T07:37:40.966 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119852.log 2024-09-14T07:37:40.967 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82023.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82831.log 2024-09-14T07:37:40.968 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.119852.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.119852.log.gz 2024-09-14T07:37:40.968 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81720.log 2024-09-14T07:37:40.969 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.88935.log 2024-09-14T07:37:40.969 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34825.log 2024-09-14T07:37:40.974 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.88935.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37279.log.gz 2024-09-14T07:37:40.978 INFO:teuthology.orchestra.run.smithi169.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.122082.log.gz 2024-09-14T07:37:40.978 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118546.log 2024-09-14T07:37:40.979 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80710.log 2024-09-14T07:37:40.983 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118546.log: 87.9% 53.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35419.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.30879.log.gz 2024-09-14T07:37:40.983 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:40.983 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52167.log.gz 2024-09-14T07:37:40.983 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80978.log.gz 2024-09-14T07:37:40.983 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43654.log.gz 2024-09-14T07:37:40.984 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118546.log.gz 2024-09-14T07:37:40.984 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121818.log 2024-09-14T07:37:40.984 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67092.log.gz 2024-09-14T07:37:40.984 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63890.log.gz 2024-09-14T07:37:40.984 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125735.log 2024-09-14T07:37:40.985 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121818.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121818.log.gz 2024-09-14T07:37:40.985 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66941.log.gz 2024-09-14T07:37:40.986 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87718.log 2024-09-14T07:37:40.987 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125735.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125735.log.gz 2024-09-14T07:37:40.987 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75510.log 2024-09-14T07:37:40.988 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75417.log.gz 2024-09-14T07:37:40.988 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87718.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87718.log.gz 2024-09-14T07:37:40.988 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74360.log 2024-09-14T07:37:40.989 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103823.log 2024-09-14T07:37:40.990 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77281.log.gz 2024-09-14T07:37:40.991 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74360.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77688.log.gz 2024-09-14T07:37:40.991 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121932.log 2024-09-14T07:37:40.993 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.103823.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103823.log.gz 2024-09-14T07:37:40.994 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76665.log 2024-09-14T07:37:40.994 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121932.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121932.log.gz 2024-09-14T07:37:40.995 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45617.log 2024-09-14T07:37:40.996 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45724.log 2024-09-14T07:37:40.996 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57556.log.gz 2024-09-14T07:37:40.997 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45617.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82190.log 2024-09-14T07:37:40.997 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90110.log 2024-09-14T07:37:40.998 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82190.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106029.log 2024-09-14T07:37:40.999 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90110.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78651.log.gz 2024-09-14T07:37:40.999 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78752.log.gz 2024-09-14T07:37:41.001 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.131303.log.gz 2024-09-14T07:37:41.001 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82831.log.gz 2024-09-14T07:37:41.001 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125982.log 2024-09-14T07:37:41.002 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106029.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.106029.log.gz 2024-09-14T07:37:41.002 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34825.log.gz 2024-09-14T07:37:41.002 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70177.log 2024-09-14T07:37:41.003 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80710.log.gz 2024-09-14T07:37:41.003 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125982.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125982.log.gz 2024-09-14T07:37:41.004 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59537.log 2024-09-14T07:37:41.004 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62123.log 2024-09-14T07:37:41.005 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52446.log 2024-09-14T07:37:41.006 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62123.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84753.log.gz 2024-09-14T07:37:41.008 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50993.log 2024-09-14T07:37:41.009 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108429.log 2024-09-14T07:37:41.009 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50993.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81518.log.gz 2024-09-14T07:37:41.009 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81584.log.gz 2024-09-14T07:37:41.010 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118638.log 2024-09-14T07:37:41.012 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.108429.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.108429.log.gz 2024-09-14T07:37:41.013 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83841.log 2024-09-14T07:37:41.013 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.88935.log.gz 2024-09-14T07:37:41.013 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71665.log 2024-09-14T07:37:41.014 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46955.log 2024-09-14T07:37:41.014 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69262.log 2024-09-14T07:37:41.015 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53356.log 2024-09-14T07:37:41.016 INFO:teuthology.orchestra.run.smithi169.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118638.log.gz 2024-09-14T07:37:41.016 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69262.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45263.log 2024-09-14T07:37:41.016 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72502.log 2024-09-14T07:37:41.019 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53356.log: /var/log/ceph/cluster2-client.mirror.45263.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50121.log.gz 2024-09-14T07:37:41.019 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120346.log 2024-09-14T07:37:41.020 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114035.log 2024-09-14T07:37:41.020 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72502.log: /var/log/ceph/cluster1-client.mirror.120346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109003.log 2024-09-14T07:37:41.021 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114035.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51051.log.gz 2024-09-14T07:37:41.021 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.120346.log.gz 2024-09-14T07:37:41.021 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114035.log.gz 2024-09-14T07:37:41.023 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64634.log 2024-09-14T07:37:41.023 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51516.log.gz 2024-09-14T07:37:41.023 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109003.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65784.log.gz 2024-09-14T07:37:41.024 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66005.log.gz 2024-09-14T07:37:41.024 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35012.log 2024-09-14T07:37:41.025 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64634.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46804.log.gz 2024-09-14T07:37:41.027 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49935.log.gz 2024-09-14T07:37:41.029 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76630.log.gz 2024-09-14T07:37:41.029 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76572.log.gz 2024-09-14T07:37:41.029 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76665.log.gz 2024-09-14T07:37:41.030 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76758.log.gz 2024-09-14T07:37:41.030 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61189.log 2024-09-14T07:37:41.030 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35012.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76909.log.gz 2024-09-14T07:37:41.031 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77188.log.gz 2024-09-14T07:37:41.031 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77223.log.gz 2024-09-14T07:37:41.031 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93182.log 2024-09-14T07:37:41.032 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61189.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50993.log.gz 2024-09-14T07:37:41.032 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81720.log.gz 2024-09-14T07:37:41.033 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48850.log.gz 2024-09-14T07:37:41.034 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82023.log.gz 2024-09-14T07:37:41.035 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.116183.log.gz 2024-09-14T07:37:41.035 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72502.log.gz 2024-09-14T07:37:41.035 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54731.log 2024-09-14T07:37:41.036 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95013.log 2024-09-14T07:37:41.036 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93182.log: /var/log/ceph/cluster2-client.mirror.54731.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93182.log.gz 2024-09-14T07:37:41.037 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47792.log 2024-09-14T07:37:41.037 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134455.log 2024-09-14T07:37:41.038 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95013.log: /var/log/ceph/cluster2-client.mirror.47792.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95013.log.gz 88.8% 2024-09-14T07:37:41.038 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.75510.log.gz 2024-09-14T07:37:41.038 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32496.log 2024-09-14T07:37:41.039 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53279.log 2024-09-14T07:37:41.039 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134455.log: /var/log/ceph/cluster2-client.mirror.32496.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134455.log.gz 2024-09-14T07:37:41.040 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130234.log 2024-09-14T07:37:41.040 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43747.log 2024-09-14T07:37:41.041 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53279.log: /var/log/ceph/cluster1-client.mirror.130234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73523.log 2024-09-14T07:37:41.041 INFO:teuthology.orchestra.run.smithi169.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.130234.log.gz 2024-09-14T07:37:41.041 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82190.log.gz 2024-09-14T07:37:41.041 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43747.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53539.log 2024-09-14T07:37:41.041 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37337.log 2024-09-14T07:37:41.042 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73523.log: /var/log/ceph/cluster2-client.mirror.53539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76200.log 2024-09-14T07:37:41.042 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30355.log 2024-09-14T07:37:41.043 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37337.log: /var/log/ceph/cluster2-client.mirror.76200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58765.log 2024-09-14T07:37:41.043 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73767.log 2024-09-14T07:37:41.043 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30355.log: /var/log/ceph/cluster2-client.mirror.58765.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30355.log.gz 2024-09-14T07:37:41.044 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.132073.log 2024-09-14T07:37:41.044 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85359.log.gz 2024-09-14T07:37:41.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68985.log 2024-09-14T07:37:41.045 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127808.log 2024-09-14T07:37:41.047 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.132073.log: /var/log/ceph/cluster2-client.mirror.68985.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84311.log.gz 2024-09-14T07:37:41.048 INFO:teuthology.orchestra.run.smithi169.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.132073.log.gz 2024-09-14T07:37:41.049 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.32496.log.gz 2024-09-14T07:37:41.050 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83841.log.gz 2024-09-14T07:37:41.050 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119361.log 2024-09-14T07:37:41.051 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68943.log 2024-09-14T07:37:41.051 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127808.log: 86.7% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71665.log.gz 2024-09-14T07:37:41.051 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.127808.log.gz 2024-09-14T07:37:41.052 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.119361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99967.log 2024-09-14T07:37:41.052 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68943.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59846.log 2024-09-14T07:37:41.053 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61189.log.gz 2024-09-14T07:37:41.053 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.107719.log.gz 2024-09-14T07:37:41.054 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52446.log.gz 2024-09-14T07:37:41.054 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114825.log 2024-09-14T07:37:41.054 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.119361.log.gz 2024-09-14T07:37:41.054 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.59846.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64634.log.gz 2024-09-14T07:37:41.054 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49598.log 2024-09-14T07:37:41.055 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51365.log 2024-09-14T07:37:41.056 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.114825.log: /var/log/ceph/cluster2-client.mirror.49598.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35012.log.gz 2024-09-14T07:37:41.056 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37337.log.gz 2024-09-14T07:37:41.057 INFO:teuthology.orchestra.run.smithi169.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.114825.log.gz 2024-09-14T07:37:41.057 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105658.log 2024-09-14T07:37:41.057 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43747.log.gz 2024-09-14T07:37:41.058 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128010.log 2024-09-14T07:37:41.058 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122872.log 2024-09-14T07:37:41.059 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.105658.log: /var/log/ceph/cluster1-client.mirror.128010.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128010.log.gz 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60503.log.gz 2024-09-14T07:37:41.059 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.060 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76200.log.gz 2024-09-14T07:37:41.060 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.105658.log.gz 2024-09-14T07:37:41.060 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41604.log 2024-09-14T07:37:41.061 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120904.log 2024-09-14T07:37:41.061 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122872.log: /var/log/ceph/cluster2-client.mirror.41604.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122872.log.gz 2024-09-14T07:37:41.062 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91311.log 2024-09-14T07:37:41.062 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51981.log 2024-09-14T07:37:41.062 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120904.log: /var/log/ceph/cluster2-client.mirror.91311.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120904.log.gz 2024-09-14T07:37:41.063 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57242.log 2024-09-14T07:37:41.063 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59537.log.gz 2024-09-14T07:37:41.064 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51981.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112777.log 2024-09-14T07:37:41.064 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36070.log 2024-09-14T07:37:41.065 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62123.log.gz 2024-09-14T07:37:41.066 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112777.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62588.log.gz 2024-09-14T07:37:41.066 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73523.log.gz 2024-09-14T07:37:41.067 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73767.log.gz 2024-09-14T07:37:41.072 INFO:teuthology.orchestra.run.smithi169.stderr: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.112777.log.gz 2024-09-14T07:37:41.074 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68943.log.gz 2024-09-14T07:37:41.074 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49598.log.gz 2024-09-14T07:37:41.074 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51365.log.gz 2024-09-14T07:37:41.080 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65099.log 2024-09-14T07:37:41.081 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41697.log 2024-09-14T07:37:41.081 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80473.log.gz 2024-09-14T07:37:41.081 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65099.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87402.log.gz 2024-09-14T07:37:41.082 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109285.log 2024-09-14T07:37:41.083 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102636.log 2024-09-14T07:37:41.084 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109285.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109285.log.gz 2024-09-14T07:37:41.088 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68985.log.gz 2024-09-14T07:37:41.088 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51981.log.gz 2024-09-14T07:37:41.092 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105092.log 2024-09-14T07:37:41.094 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102636.log: 73.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45724.log.gz 2024-09-14T07:37:41.094 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46955.log.gz 2024-09-14T07:37:41.095 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47269.log.gz 2024-09-14T07:37:41.095 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47362.log.gz 2024-09-14T07:37:41.095 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37430.log 2024-09-14T07:37:41.096 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47792.log.gz 2024-09-14T07:37:41.098 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.105092.log: 96.1% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53279.log.gz 2024-09-14T07:37:41.098 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53356.log.gz 2024-09-14T07:37:41.098 INFO:teuthology.orchestra.run.smithi169.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.105092.log.gz 2024-09-14T07:37:41.098 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85035.log 2024-09-14T07:37:41.099 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54731.log.gz 2024-09-14T07:37:41.099 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107135.log 2024-09-14T07:37:41.100 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85035.log: -- replaced with /var/log/ceph/cluster2-client.mirror.102636.log.gz 2024-09-14T07:37:41.100 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55510.log.gz 2024-09-14T07:37:41.101 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57242.log.gz 2024-09-14T07:37:41.106 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58765.log.gz 2024-09-14T07:37:41.106 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.85101.log.gz 2024-09-14T07:37:41.107 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37430.log.gz 2024-09-14T07:37:41.107 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91840.log.gz 2024-09-14T07:37:41.109 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41604.log.gz 2024-09-14T07:37:41.110 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41697.log.gz 2024-09-14T07:37:41.112 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37151.log 2024-09-14T07:37:41.112 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107135.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125686.log 2024-09-14T07:37:41.113 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37151.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107135.log.gz 2024-09-14T07:37:41.113 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67313.log 2024-09-14T07:37:41.114 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125686.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125686.log.gz 2024-09-14T07:37:41.114 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113666.log 2024-09-14T07:37:41.115 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56870.log 2024-09-14T07:37:41.115 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116116.log 2024-09-14T07:37:41.115 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113666.log: /var/log/ceph/cluster2-client.mirror.56870.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88422.log.gz 2024-09-14T07:37:41.117 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116795.log 2024-09-14T07:37:41.117 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113666.log.gz 2024-09-14T07:37:41.117 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.116116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117611.log 2024-09-14T07:37:41.117 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87936.log 2024-09-14T07:37:41.118 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67313.log.gz 2024-09-14T07:37:41.118 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117611.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.116116.log.gz 2024-09-14T07:37:41.118 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117611.log.gz 2024-09-14T07:37:41.118 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101431.log 2024-09-14T07:37:41.121 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129293.log 2024-09-14T07:37:41.121 INFO:teuthology.orchestra.run.smithi169.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87936.log.gz 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9%/var/log/ceph/cluster2-client.mirror.101431.log: -- replaced with /var/log/ceph/cluster2-client.mirror.116795.log.gz 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112003.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%/var/log/ceph/cluster1-client.mirror.129293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63983.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.101431.log.gz 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129293.log.gz 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.112003.log: gzip 88.9% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.74360.log.gz --verbose 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: -- /var/log/ceph/cluster2-client.mirror.34732.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74453.log.gz 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86216.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97885.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.112003.log.gz/var/log/ceph/cluster2-client.mirror.86216.log: 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79090.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100552.log 2024-09-14T07:37:41.122 INFO:teuthology.orchestra.run.smithi169.stderr:gzip/var/log/ceph/cluster2-client.mirror.79090.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131378.log 2024-09-14T07:37:41.123 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97885.log.gz 2024-09-14T07:37:41.123 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68148.log 2024-09-14T07:37:41.123 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100552.log: /var/log/ceph/cluster1-client.mirror.131378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122795.log 2024-09-14T07:37:41.125 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68148.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100552.log.gz 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.131378.log.gz 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38828.log 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54359.log 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63983.log.gz 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38828.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86996.log 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.122795.log.gz 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93610.log 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84333.log 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34732.log.gz 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93610.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79090.log.gz 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132349.log 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93610.log.gz 2024-09-14T07:37:41.126 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68148.log.gz/var/log/ceph/cluster2-client.mirror.84333.log: 2024-09-14T07:37:41.127 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38084.log 2024-09-14T07:37:41.129 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64448.log 2024-09-14T07:37:41.129 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132349.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130000.log 2024-09-14T07:37:41.129 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132349.log.gz 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38084.log: /var/log/ceph/cluster2-client.mirror.64448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45340.log 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75248.log 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45340.log: 87.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109703.log 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.130000.log.gz 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.75248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63948.log 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109703.log: /var/log/ceph/cluster2-client.mirror.63948.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109703.log.gz 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60875.log 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85725.log 2024-09-14T07:37:41.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106261.log 2024-09-14T07:37:41.131 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85725.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31016.log 2024-09-14T07:37:41.131 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125560.log 2024-09-14T07:37:41.131 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31016.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59846.log.gz 2024-09-14T07:37:41.133 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.109003.log.gz 2024-09-14T07:37:41.134 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72316.log 2024-09-14T07:37:41.134 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123199.log 2024-09-14T07:37:41.134 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72316.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38828.log.gz 2024-09-14T07:37:41.135 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125560.log.gz 2024-09-14T07:37:41.135 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83503.log 2024-09-14T07:37:41.136 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.123199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40305.log 2024-09-14T07:37:41.136 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83503.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128640.log 2024-09-14T07:37:41.137 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40305.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105581.log 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84968.log 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128640.log: /var/log/ceph/cluster2-client.mirror.105581.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123199.log.gz 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128640.log.gz 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105581.log.gz 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39204.log 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55045.log 2024-09-14T07:37:41.139 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.30555.log 2024-09-14T07:37:41.140 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99436.log 2024-09-14T07:37:41.140 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55045.log: /var/log/ceph/cluster2-client.mirror.2.30555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116001.log 2024-09-14T07:37:41.140 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77746.log 2024-09-14T07:37:41.140 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116001.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99436.log.gz 2024-09-14T07:37:41.141 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.116001.log.gz 2024-09-14T07:37:41.141 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119704.log 2024-09-14T07:37:41.141 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77746.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37151.log.gz 2024-09-14T07:37:41.141 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133229.log 2024-09-14T07:37:41.141 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40305.log.gz 2024-09-14T07:37:41.142 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119704.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119704.log.gz 2024-09-14T07:37:41.142 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102164.log 2024-09-14T07:37:41.142 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133229.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130315.log 2024-09-14T07:37:41.143 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77130.log 2024-09-14T07:37:41.144 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102164.log: /var/log/ceph/cluster2-client.mirror.130315.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102164.log.gz 2024-09-14T07:37:41.144 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130315.log.gz 2024-09-14T07:37:41.146 INFO:teuthology.orchestra.run.smithi169.stderr: 55.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31016.log.gz 2024-09-14T07:37:41.146 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65099.log.gz 2024-09-14T07:37:41.148 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101944.log 2024-09-14T07:37:41.148 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79494.log 2024-09-14T07:37:41.149 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77130.log: /var/log/ceph/cluster2-client.mirror.101944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91815.log 2024-09-14T07:37:41.149 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.101944.log.gz 2024-09-14T07:37:41.149 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122904.log 2024-09-14T07:37:41.150 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79494.log: /var/log/ceph/cluster2-client.mirror.91815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79257.log 2024-09-14T07:37:41.150 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91815.log.gz 2024-09-14T07:37:41.150 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125119.log 2024-09-14T07:37:41.151 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.122904.log: /var/log/ceph/cluster2-client.mirror.79257.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82528.log 2024-09-14T07:37:41.151 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.122904.log.gz 2024-09-14T07:37:41.151 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103179.log 2024-09-14T07:37:41.152 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125119.log.gz/var/log/ceph/cluster2-client.mirror.82528.log: 2024-09-14T07:37:41.152 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68434.log 2024-09-14T07:37:41.153 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90230.log 2024-09-14T07:37:41.153 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103179.log: /var/log/ceph/cluster2-client.mirror.68434.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103179.log.gz 2024-09-14T07:37:41.153 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51609.log 2024-09-14T07:37:41.154 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90230.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90230.log.gz 2024-09-14T07:37:41.154 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36070.log.gz 2024-09-14T07:37:41.154 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99819.log 2024-09-14T07:37:41.155 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89807.log 2024-09-14T07:37:41.155 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85725.log.gz 2024-09-14T07:37:41.156 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99819.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99819.log.gz 2024-09-14T07:37:41.156 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63855.log 2024-09-14T07:37:41.156 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63948.log.gz 2024-09-14T07:37:41.157 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77746.log.gz 2024-09-14T07:37:41.157 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89807.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79257.log.gz 2024-09-14T07:37:41.158 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77130.log.gz 2024-09-14T07:37:41.161 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56870.log.gz 2024-09-14T07:37:41.162 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51609.log.gz 2024-09-14T07:37:41.162 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72316.log.gz 2024-09-14T07:37:41.162 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70363.log 2024-09-14T07:37:41.163 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55045.log.gz 2024-09-14T07:37:41.163 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31955.log 2024-09-14T07:37:41.164 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104551.log 2024-09-14T07:37:41.164 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.31955.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84902.log.gz 2024-09-14T07:37:41.165 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33461.log 2024-09-14T07:37:41.165 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104551.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104551.log.gz 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64448.log.gz 2024-09-14T07:37:41.165 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.165 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63053.log 2024-09-14T07:37:41.166 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95106.log 2024-09-14T07:37:41.167 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53987.log 2024-09-14T07:37:41.167 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95106.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95106.log.gz 2024-09-14T07:37:41.168 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88116.log 2024-09-14T07:37:41.169 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73895.log 2024-09-14T07:37:41.169 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83503.log.gz 2024-09-14T07:37:41.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88116.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86216.log.gz 2024-09-14T07:37:41.171 INFO:teuthology.orchestra.run.smithi169.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.31955.log.gz 2024-09-14T07:37:41.183 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82528.log.gz 2024-09-14T07:37:41.183 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63855.log.gz 2024-09-14T07:37:41.184 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39142.log 2024-09-14T07:37:41.191 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73895.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45340.log.gz 2024-09-14T07:37:41.192 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.45263.log.gz 2024-09-14T07:37:41.192 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104802.log 2024-09-14T07:37:41.192 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39142.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33461.log.gz 2024-09-14T07:37:41.193 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73895.log.gz 2024-09-14T07:37:41.199 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86996.log.gz 2024-09-14T07:37:41.205 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.99967.log.gz 2024-09-14T07:37:41.206 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132050.log 2024-09-14T07:37:41.206 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104802.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104802.log.gz 2024-09-14T07:37:41.207 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39576.log 2024-09-14T07:37:41.208 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132050.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38084.log.gz 2024-09-14T07:37:41.210 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39142.log.gz 2024-09-14T07:37:41.210 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39204.log.gz 2024-09-14T07:37:41.213 INFO:teuthology.orchestra.run.smithi169.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.2.30555.log.gz 2024-09-14T07:37:41.220 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69020.log 2024-09-14T07:37:41.220 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84968.log.gz 2024-09-14T07:37:41.221 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39576.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39576.log.gz 2024-09-14T07:37:41.226 INFO:teuthology.orchestra.run.smithi169.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84333.log.gz 2024-09-14T07:37:41.227 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85035.log.gz 2024-09-14T07:37:41.229 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70177.log.gz 2024-09-14T07:37:41.229 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70363.log.gz 2024-09-14T07:37:41.236 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45484.log 2024-09-14T07:37:41.237 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69020.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69020.log.gz 2024-09-14T07:37:41.237 INFO:teuthology.orchestra.run.smithi169.stderr: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59868.log.gz 2024-09-14T07:37:41.238 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.75248.log.gz 2024-09-14T07:37:41.243 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60875.log.gz 2024-09-14T07:37:41.244 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130649.log 2024-09-14T07:37:41.246 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.45484.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46583.log.gz 2024-09-14T07:37:41.247 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53987.log.gz 2024-09-14T07:37:41.248 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54359.log.gz 2024-09-14T07:37:41.248 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.45484.log.gz 2024-09-14T07:37:41.252 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63053.log.gz 2024-09-14T07:37:41.256 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79494.log.gz 2024-09-14T07:37:41.269 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132600.log 2024-09-14T07:37:41.269 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7%/var/log/ceph/cluster1-client.mirror.130649.log: -- replaced with /var/log/ceph/cluster2-client.mirror.90253.log.gz 2024-09-14T07:37:41.269 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130649.log.gz 2024-09-14T07:37:41.269 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126540.log 2024-09-14T07:37:41.270 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132600.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.132600.log.gz 2024-09-14T07:37:41.270 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96183.log 2024-09-14T07:37:41.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126540.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126540.log.gz 2024-09-14T07:37:41.272 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131153.log 2024-09-14T07:37:41.275 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96183.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.96183.log.gz 2024-09-14T07:37:41.275 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60689.log 2024-09-14T07:37:41.275 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90110.log.gz 2024-09-14T07:37:41.276 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131153.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131153.log.gz 2024-09-14T07:37:41.276 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69712.log 2024-09-14T07:37:41.276 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42166.log 2024-09-14T07:37:41.277 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33612.log 2024-09-14T07:37:41.277 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69712.log: /var/log/ceph/cluster2-client.mirror.42166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103746.log 2024-09-14T07:37:41.278 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125303.log 2024-09-14T07:37:41.278 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33612.log: /var/log/ceph/cluster2-client.mirror.103746.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103746.log.gz 2024-09-14T07:37:41.278 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69712.log.gz 2024-09-14T07:37:41.279 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61809.log 2024-09-14T07:37:41.280 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33612.log.gz 2024-09-14T07:37:41.280 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125303.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125303.log.gz 2024-09-14T07:37:41.280 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30845.log 2024-09-14T07:37:41.280 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130418.log 2024-09-14T07:37:41.281 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61809.log: /var/log/ceph/cluster1-client.mirror.30845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103845.log 2024-09-14T07:37:41.281 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61340.log 2024-09-14T07:37:41.281 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8%/var/log/ceph/cluster2-client.mirror.103845.log: -- replaced with /var/log/ceph/cluster2-client.mirror.61809.log.gz 2024-09-14T07:37:41.282 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130418.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.130418.log.gz 2024-09-14T07:37:41.283 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57056.log 2024-09-14T07:37:41.283 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61340.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119406.log 2024-09-14T07:37:41.284 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57056.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53673.log 2024-09-14T07:37:41.285 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61340.log.gz 2024-09-14T07:37:41.285 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57056.log.gz 2024-09-14T07:37:41.287 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.119406.log: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.119406.log.gz 2024-09-14T07:37:41.288 INFO:teuthology.orchestra.run.smithi169.stderr: 63.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30845.log.gz 2024-09-14T07:37:41.292 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91564.log 2024-09-14T07:37:41.293 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131215.log 2024-09-14T07:37:41.295 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91564.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.106261.log.gz 2024-09-14T07:37:41.296 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101994.log 2024-09-14T07:37:41.301 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131215.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131215.log.gz 2024-09-14T07:37:41.305 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33089.log 2024-09-14T07:37:41.305 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101994.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101994.log.gz 2024-09-14T07:37:41.306 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94136.log 2024-09-14T07:37:41.309 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33089.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32961.log 2024-09-14T07:37:41.310 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94136.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42166.log.gz 2024-09-14T07:37:41.324 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48722.log 2024-09-14T07:37:41.332 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32961.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35326.log 2024-09-14T07:37:41.340 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130051.log 2024-09-14T07:37:41.349 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35326.log: 95.9% 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35326.log.gz 2024-09-14T07:37:41.349 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.132050.log.gz 2024-09-14T07:37:41.359 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48722.log.gz 2024-09-14T07:37:41.360 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60317.log 2024-09-14T07:37:41.360 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94696.log 2024-09-14T07:37:41.361 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130051.log: /var/log/ceph/cluster2-client.mirror.60317.log: 85.4% -- replaced with /var/log/ceph/cluster2-client.mirror.130051.log.gz 2024-09-14T07:37:41.364 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32961.log.gz 2024-09-14T07:37:41.364 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33089.log.gz 2024-09-14T07:37:41.364 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33240.log.gz 2024-09-14T07:37:41.367 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60689.log.gz 2024-09-14T07:37:41.368 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65912.log 2024-09-14T07:37:41.369 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94696.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.94696.log.gz 2024-09-14T07:37:41.369 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83301.log 2024-09-14T07:37:41.369 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68299.log 2024-09-14T07:37:41.370 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84017.log 2024-09-14T07:37:41.371 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68299.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110600.log 2024-09-14T07:37:41.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34360.log 2024-09-14T07:37:41.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110600.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110600.log.gz 2024-09-14T07:37:41.375 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65912.log.gz 2024-09-14T07:37:41.376 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68299.log.gz 2024-09-14T07:37:41.376 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93544.log 2024-09-14T07:37:41.377 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53314.log 2024-09-14T07:37:41.380 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93544.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34360.log.gz 2024-09-14T07:37:41.381 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93544.log.gz 2024-09-14T07:37:41.381 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126966.log 2024-09-14T07:37:41.382 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53314.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92728.log 2024-09-14T07:37:41.385 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126966.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53314.log.gz 2024-09-14T07:37:41.386 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126966.log.gz 2024-09-14T07:37:41.386 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119897.log 2024-09-14T07:37:41.389 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.92728.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92728.log.gz 2024-09-14T07:37:41.390 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93302.log 2024-09-14T07:37:41.395 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.119897.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.119897.log.gz 2024-09-14T07:37:41.395 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91409.log 2024-09-14T07:37:41.396 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93302.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93302.log.gz 2024-09-14T07:37:41.396 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48350.log 2024-09-14T07:37:41.404 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50528.log 2024-09-14T07:37:41.408 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48350.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48350.log.gz 2024-09-14T07:37:41.411 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91311.log.gz 2024-09-14T07:37:41.413 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114661.log 2024-09-14T07:37:41.414 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37712.log 2024-09-14T07:37:41.415 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114661.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114661.log.gz 2024-09-14T07:37:41.415 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102917.log 2024-09-14T07:37:41.422 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37712.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50528.log.gz 2024-09-14T07:37:41.423 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37712.log.gz 2024-09-14T07:37:41.423 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103975.log 2024-09-14T07:37:41.424 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.102917.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.102917.log.gz 2024-09-14T07:37:41.424 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84017.log.gz 2024-09-14T07:37:41.425 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78348.log 2024-09-14T07:37:41.426 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103975.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103975.log.gz 2024-09-14T07:37:41.426 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124399.log 2024-09-14T07:37:41.427 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66813.log 2024-09-14T07:37:41.428 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124399.log: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.91564.log.gz 2024-09-14T07:37:41.428 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.124399.log.gz 2024-09-14T07:37:41.428 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85956.log 2024-09-14T07:37:41.429 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73953.log 2024-09-14T07:37:41.429 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85956.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51551.log 2024-09-14T07:37:41.429 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65691.log 2024-09-14T07:37:41.430 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51551.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34267.log 2024-09-14T07:37:41.430 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65691.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119658.log 2024-09-14T07:37:41.431 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34267.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66813.log.gz 2024-09-14T07:37:41.432 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30380.log 2024-09-14T07:37:41.432 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119658.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89807.log.gz 2024-09-14T07:37:41.432 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.119658.log.gz 2024-09-14T07:37:41.433 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71758.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73953.log.gz 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34546.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30380.log.gz 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71758.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41139.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127785.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34546.log: /var/log/ceph/cluster2-client.mirror.41139.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83639.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97546.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127785.log.gz 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74980.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54487.log 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51551.log.gz 2024-09-14T07:37:41.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74980.log: /var/log/ceph/cluster1-client.mirror.97546.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34267.log.gz 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45643.log 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57707.log 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65691.log.gz 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.45643.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.97546.log.gz 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59400.log 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55231.log 2024-09-14T07:37:41.439 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131174.log 2024-09-14T07:37:41.440 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41139.log.gz 2024-09-14T07:37:41.440 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65176.log 2024-09-14T07:37:41.440 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41104.log 2024-09-14T07:37:41.440 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.65176.log: 88.1%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.131174.log.gz 2024-09-14T07:37:41.440 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122297.log 2024-09-14T07:37:41.441 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41104.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34546.log.gz 2024-09-14T07:37:41.442 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74980.log.gz 2024-09-14T07:37:41.442 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2024-09-14T07:37:41.443 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.122297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70735.log 2024-09-14T07:37:41.443 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129045.log 2024-09-14T07:37:41.443 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44921.log 2024-09-14T07:37:41.447 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118154.log 2024-09-14T07:37:41.447 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.129045.log.gz 2024-09-14T07:37:41.447 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122297.log.gz 2024-09-14T07:37:41.447 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54115.log 2024-09-14T07:37:41.447 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44921.log: /var/log/ceph/cluster2-client.mirror.118154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133100.log 2024-09-14T07:37:41.447 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.118154.log.gz/var/log/ceph/cluster2-client.mirror.54115.log: 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57707.log.gz 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133797.log 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60910.log 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133797.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133100.log.gz 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133797.log.gz 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61565.log 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr:gzip/var/log/ceph/cluster2-client.mirror.60910.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45375.log 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59400.log.gz 2024-09-14T07:37:41.448 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61565.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55231.log.gz 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127079.log 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45375.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41104.log.gz 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132879.log 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.127079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55324.log 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132879.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.45643.log.gz 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.132879.log.gz 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44921.log.gz 2024-09-14T07:37:41.450 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127079.log.gz 2024-09-14T07:37:41.451 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54115.log.gz 2024-09-14T07:37:41.452 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61565.log.gz 2024-09-14T07:37:41.452 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118028.log 2024-09-14T07:37:41.452 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55324.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48629.log 2024-09-14T07:37:41.452 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.118028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106373.log 2024-09-14T07:37:41.452 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48629.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.45375.log.gz 2024-09-14T07:37:41.452 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55324.log.gz 2024-09-14T07:37:41.457 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48629.log.gz 2024-09-14T07:37:41.457 INFO:teuthology.orchestra.run.smithi169.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.118028.log.gz 2024-09-14T07:37:41.457 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60910.log.gz 2024-09-14T07:37:41.457 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76537.log 2024-09-14T07:37:41.457 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106373.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75917.log 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76537.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.106373.log.gz 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99471.log 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110286.log 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80306.log 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70735.log.gz 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113469.log 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99471.log: /var/log/ceph/cluster2-client.mirror.80306.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76537.log.gz 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99471.log.gz 2024-09-14T07:37:41.458 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110286.log.gz 2024-09-14T07:37:41.459 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80306.log.gz 2024-09-14T07:37:41.459 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78348.log.gz 2024-09-14T07:37:41.460 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70549.log 2024-09-14T07:37:41.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113469.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.30505.log 2024-09-14T07:37:41.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70549.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132196.log 2024-09-14T07:37:41.461 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.113469.log.gz 2024-09-14T07:37:41.461 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115192.log 2024-09-14T07:37:41.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.30505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89910.log 2024-09-14T07:37:41.462 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132196.log: /var/log/ceph/cluster2-client.mirror.115192.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.132196.log.gz 2024-09-14T07:37:41.462 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129509.log 2024-09-14T07:37:41.462 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115192.log.gz 2024-09-14T07:37:41.463 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89910.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70549.log.gz 2024-09-14T07:37:41.463 INFO:teuthology.orchestra.run.smithi169.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89910.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104702.log 2024-09-14T07:37:41.463 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.463 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.129509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48664.log 2024-09-14T07:37:41.464 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104702.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83639.log.gz 2024-09-14T07:37:41.466 INFO:teuthology.orchestra.run.smithi129.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2024-09-14T07:37:41.468 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60317.log.gz 2024-09-14T07:37:41.470 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88116.log.gz 2024-09-14T07:37:41.473 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133229.log.gz 2024-09-14T07:37:41.473 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95586.log 2024-09-14T07:37:41.473 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91166.log 2024-09-14T07:37:41.473 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48664.log: /var/log/ceph/cluster2-client.mirror.95586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48106.log 2024-09-14T07:37:41.474 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54266.log 2024-09-14T07:37:41.475 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48106.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108352.log 2024-09-14T07:37:41.475 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85445.log 2024-09-14T07:37:41.477 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108352.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48664.log.gz 2024-09-14T07:37:41.479 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54266.log.gz 2024-09-14T07:37:41.479 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108352.log.gz 2024-09-14T07:37:41.479 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53202.log 2024-09-14T07:37:41.480 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85445.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69320.log 2024-09-14T07:37:41.480 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53202.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75917.log.gz 2024-09-14T07:37:41.481 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54545.log 2024-09-14T07:37:41.481 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.69320.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120981.log 2024-09-14T07:37:41.482 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54545.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53202.log.gz 2024-09-14T07:37:41.483 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.69320.log.gz 2024-09-14T07:37:41.483 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66191.log 2024-09-14T07:37:41.487 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.120981.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.120981.log.gz 2024-09-14T07:37:41.487 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75545.log 2024-09-14T07:37:41.487 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85956.log.gz 2024-09-14T07:37:41.488 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84988.log 2024-09-14T07:37:41.488 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85767.log 2024-09-14T07:37:41.489 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66191.log.gz 2024-09-14T07:37:41.489 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75545.log.gz 2024-09-14T07:37:41.490 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80877.log 2024-09-14T07:37:41.490 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109784.log 2024-09-14T07:37:41.494 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60817.log 2024-09-14T07:37:41.494 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54487.log.gz 2024-09-14T07:37:41.494 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54545.log.gz 2024-09-14T07:37:41.495 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109784.log: 88.0% 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80877.log.gz 2024-09-14T07:37:41.496 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.109784.log.gz 2024-09-14T07:37:41.496 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124123.log 2024-09-14T07:37:41.497 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123247.log 2024-09-14T07:37:41.499 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124123.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48106.log.gz 2024-09-14T07:37:41.501 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85767.log.gz 2024-09-14T07:37:41.502 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124123.log.gz 2024-09-14T07:37:41.502 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71165.log 2024-09-14T07:37:41.503 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123247.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123247.log.gz 2024-09-14T07:37:41.503 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80100.log 2024-09-14T07:37:41.512 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64227.log 2024-09-14T07:37:41.512 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123318.log 2024-09-14T07:37:41.513 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114440.log 2024-09-14T07:37:41.514 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123318.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.123318.log.gz 2024-09-14T07:37:41.518 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64227.log.gz 2024-09-14T07:37:41.518 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124644.log 2024-09-14T07:37:41.520 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114440.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85445.log.gz 2024-09-14T07:37:41.522 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80100.log.gz 2024-09-14T07:37:41.532 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104111.log 2024-09-14T07:37:41.533 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124644.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124644.log.gz 2024-09-14T07:37:41.546 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103845.log.gz 2024-09-14T07:37:41.548 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61433.log 2024-09-14T07:37:41.548 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104111.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104111.log.gz 2024-09-14T07:37:41.560 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.104702.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42352.log 2024-09-14T07:37:41.561 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.564 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61433.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61433.log.gz 2024-09-14T07:37:41.574 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84988.log.gz 2024-09-14T07:37:41.576 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58951.log 2024-09-14T07:37:41.577 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124541.log 2024-09-14T07:37:41.578 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88677.log 2024-09-14T07:37:41.578 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58951.log.gz 2024-09-14T07:37:41.579 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124541.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124541.log.gz 2024-09-14T07:37:41.579 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42352.log.gz 2024-09-14T07:37:41.580 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37558.log 2024-09-14T07:37:41.580 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82932.log 2024-09-14T07:37:41.581 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88677.log: /var/log/ceph/cluster2-client.mirror.37558.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88677.log.gz 2024-09-14T07:37:41.581 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65156.log 2024-09-14T07:37:41.581 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123403.log 2024-09-14T07:37:41.582 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73802.log 2024-09-14T07:37:41.582 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125877.log 2024-09-14T07:37:41.583 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82932.log: /var/log/ceph/cluster2-client.mirror.123403.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37558.log.gz 2024-09-14T07:37:41.584 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65156.log: /var/log/ceph/cluster2-client.mirror.73802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103498.log 2024-09-14T07:37:41.585 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125877.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.123403.log.gz 2024-09-14T07:37:41.585 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125877.log.gz 2024-09-14T07:37:41.586 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134249.log 2024-09-14T07:37:41.586 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103498.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103498.log.gz 2024-09-14T07:37:41.587 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46025.log 2024-09-14T07:37:41.587 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134249.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134249.log.gz 2024-09-14T07:37:41.588 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110015.log 2024-09-14T07:37:41.588 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46025.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123361.log 2024-09-14T07:37:41.589 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133407.log 2024-09-14T07:37:41.589 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110015.log: /var/log/ceph/cluster2-client.mirror.123361.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110015.log.gz 2024-09-14T07:37:41.590 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46397.log 2024-09-14T07:37:41.590 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123361.log.gz 2024-09-14T07:37:41.590 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80811.log 2024-09-14T07:37:41.591 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46397.log: 88.4% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95586.log.gz 2024-09-14T07:37:41.591 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.133407.log.gz 2024-09-14T07:37:41.591 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118658.log 2024-09-14T07:37:41.591 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33519.log 2024-09-14T07:37:41.592 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40481.log 2024-09-14T07:37:41.592 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.118658.log: /var/log/ceph/cluster2-client.mirror.33519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128955.log 2024-09-14T07:37:41.593 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40481.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73802.log.gz 2024-09-14T07:37:41.593 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.46025.log.gz -5 2024-09-14T07:37:41.593 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.110754.log 2024-09-14T07:37:41.594 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0%/var/log/ceph/cluster1-client.mirror.128955.log: -- replaced with /var/log/ceph/cluster1-client.mirror.118658.log.gz 2024-09-14T07:37:41.594 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2024-09-14T07:37:41.594 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63332.log 2024-09-14T07:37:41.595 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.110754.log: /var/log/ceph/cluster2-mon.a.log: -- replaced with /var/log/ceph/cluster1-client.mirror.128955.log.gz 2024-09-14T07:37:41.595 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127628.log 2024-09-14T07:37:41.595 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99528.log 2024-09-14T07:37:41.596 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.46397.log.gz 2024-09-14T07:37:41.596 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127628.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127628.log.gz 2024-09-14T07:37:41.597 INFO:teuthology.orchestra.run.smithi169.stderr: 91.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92911.log 2024-09-14T07:37:41.597 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.110754.log.gz 2024-09-14T07:37:41.597 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132773.log 2024-09-14T07:37:41.597 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80811.log.gz 2024-09-14T07:37:41.597 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.92911.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.92911.log.gz 2024-09-14T07:37:41.599 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32903.log 2024-09-14T07:37:41.599 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.99528.log.gzgzip 2024-09-14T07:37:41.599 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63111.log 2024-09-14T07:37:41.599 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132773.log: /var/log/ceph/cluster2-client.mirror.32903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31140.log 2024-09-14T07:37:41.599 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63111.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132773.log.gz 2024-09-14T07:37:41.600 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55696.log 2024-09-14T07:37:41.600 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104246.log 2024-09-14T07:37:41.602 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72002.log 2024-09-14T07:37:41.602 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55696.log: /var/log/ceph/cluster2-client.mirror.104246.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33519.log.gz 2024-09-14T07:37:41.602 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31811.log 2024-09-14T07:37:41.602 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.104246.log.gz 2024-09-14T07:37:41.602 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56591.log 2024-09-14T07:37:41.602 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38026.log 2024-09-14T07:37:41.604 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122252.log 2024-09-14T07:37:41.604 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56591.log: /var/log/ceph/cluster2-client.mirror.38026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30255.log 2024-09-14T07:37:41.605 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.122252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107234.log 2024-09-14T07:37:41.605 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.122252.log.gz 2024-09-14T07:37:41.605 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106747.log 2024-09-14T07:37:41.605 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30255.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30255.log.gz 2024-09-14T07:37:41.605 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43933.log 2024-09-14T07:37:41.606 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106747.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106747.log.gz 2024-09-14T07:37:41.606 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43282.log 2024-09-14T07:37:41.606 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77316.log 2024-09-14T07:37:41.607 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43933.log: /var/log/ceph/cluster2-client.mirror.43282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58579.log 2024-09-14T07:37:41.607 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74232.log 2024-09-14T07:37:41.608 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106938.log 2024-09-14T07:37:41.608 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58579.log: /var/log/ceph/cluster2-client.mirror.74232.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47920.log 2024-09-14T07:37:41.608 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106938.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30555.log 2024-09-14T07:37:41.609 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106938.log.gz/var/log/ceph/cluster2-client.mirror.47920.log: 2024-09-14T07:37:41.612 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108869.log 2024-09-14T07:37:41.612 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30555.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59933.log 2024-09-14T07:37:41.613 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100264.log 2024-09-14T07:37:41.613 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108869.log: /var/log/ceph/cluster1-client.admin.59933.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58579.log.gz 2024-09-14T07:37:41.613 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108869.log.gz 2024-09-14T07:37:41.614 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30405.log 2024-09-14T07:37:41.615 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100264.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100264.log.gz 2024-09-14T07:37:41.615 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113841.log 2024-09-14T07:37:41.616 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.30405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118448.log 2024-09-14T07:37:41.616 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99570.log 2024-09-14T07:37:41.618 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113841.log: 88.2%/var/log/ceph/cluster2-client.mirror.118448.log: -- replaced with /var/log/ceph/cluster2-client.mirror.113841.log.gz 2024-09-14T07:37:41.618 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.40613.log 2024-09-14T07:37:41.618 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.118448.log.gz 2024-09-14T07:37:41.618 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35105.log 2024-09-14T07:37:41.619 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47920.log.gz 2024-09-14T07:37:41.621 INFO:teuthology.orchestra.run.smithi169.stderr: 80.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30555.log.gz 2024-09-14T07:37:41.621 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31140.log.gz 2024-09-14T07:37:41.621 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63111.log.gz 2024-09-14T07:37:41.622 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.38026.log.gz 2024-09-14T07:37:41.622 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32903.log.gz 2024-09-14T07:37:41.622 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43282.log.gz 2024-09-14T07:37:41.622 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43933.log.gz 2024-09-14T07:37:41.623 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.40613.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56591.log.gz 2024-09-14T07:37:41.625 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77316.log.gz 2024-09-14T07:37:41.625 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110411.log 2024-09-14T07:37:41.626 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35105.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60817.log.gz 2024-09-14T07:37:41.626 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87425.log 2024-09-14T07:37:41.627 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110411.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110411.log.gz 2024-09-14T07:37:41.627 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70514.log 2024-09-14T07:37:41.628 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87425.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58172.log 2024-09-14T07:37:41.629 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70514.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.99570.log.gz 2024-09-14T07:37:41.629 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112048.log 2024-09-14T07:37:41.629 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58544.log 2024-09-14T07:37:41.630 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58172.log: /var/log/ceph/cluster1-client.mirror.112048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127155.log 2024-09-14T07:37:41.630 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105241.log 2024-09-14T07:37:41.631 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127155.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127155.log.gz 2024-09-14T07:37:41.631 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.112048.log.gz 2024-09-14T07:37:41.632 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105465.log 2024-09-14T07:37:41.633 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124561.log 2024-09-14T07:37:41.633 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105465.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.105241.log.gz 2024-09-14T07:37:41.633 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127605.log 2024-09-14T07:37:41.633 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105465.log.gz 2024-09-14T07:37:41.634 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128033.log 2024-09-14T07:37:41.634 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124561.log: /var/log/ceph/cluster1-client.mirror.127605.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127605.log.gz 2024-09-14T07:37:41.635 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52955.log 2024-09-14T07:37:41.635 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128033.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128033.log.gz 2024-09-14T07:37:41.635 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98859.log 2024-09-14T07:37:41.636 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52955.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124561.log.gz 2024-09-14T07:37:41.638 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.91867.log 2024-09-14T07:37:41.638 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98859.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48571.log 2024-09-14T07:37:41.640 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.1.91867.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98859.log.gz 2024-09-14T07:37:41.640 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55696.log.gz 2024-09-14T07:37:41.640 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49094.log 2024-09-14T07:37:41.641 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114070.log 2024-09-14T07:37:41.641 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49094.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128055.log 2024-09-14T07:37:41.642 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114070.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114070.log.gz 2024-09-14T07:37:41.643 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105994.log 2024-09-14T07:37:41.643 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128055.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.128055.log.gz 2024-09-14T07:37:41.644 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73674.log 2024-09-14T07:37:41.644 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105994.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63332.log.gz 2024-09-14T07:37:41.645 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105994.log.gz 2024-09-14T07:37:41.648 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32589.log 2024-09-14T07:37:41.648 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73674.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112941.log 2024-09-14T07:37:41.649 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117870.log 2024-09-14T07:37:41.650 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.112941.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71165.log.gz 2024-09-14T07:37:41.651 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71758.log.gz 2024-09-14T07:37:41.652 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72002.log.gz 2024-09-14T07:37:41.652 INFO:teuthology.orchestra.run.smithi169.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.112941.log.gz 2024-09-14T07:37:41.653 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.114440.log.gz 2024-09-14T07:37:41.653 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88492.log 2024-09-14T07:37:41.653 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60445.log 2024-09-14T07:37:41.654 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117870.log: /var/log/ceph/cluster2-client.mirror.88492.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117870.log.gz 2024-09-14T07:37:41.665 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58172.log.gz 2024-09-14T07:37:41.666 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48571.log.gz 2024-09-14T07:37:41.666 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70514.log.gz 2024-09-14T07:37:41.668 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32682.log 2024-09-14T07:37:41.672 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60445.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73674.log.gz 2024-09-14T07:37:41.674 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58544.log.gz 2024-09-14T07:37:41.675 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52955.log.gz 2024-09-14T07:37:41.675 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32589.log.gz 2024-09-14T07:37:41.675 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60445.log.gz 2024-09-14T07:37:41.677 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83301.log.gz 2024-09-14T07:37:41.680 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95169.log 2024-09-14T07:37:41.681 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32682.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.40613.log.gz 2024-09-14T07:37:41.682 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34081.log 2024-09-14T07:37:41.683 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95169.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95169.log.gz 2024-09-14T07:37:41.683 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30760.log 2024-09-14T07:37:41.683 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133203.log 2024-09-14T07:37:41.685 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30760.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32682.log.gz 2024-09-14T07:37:41.686 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.34081.log.gz 2024-09-14T07:37:41.687 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.59933.log.gz 2024-09-14T07:37:41.687 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97044.log 2024-09-14T07:37:41.687 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133203.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133203.log.gz 2024-09-14T07:37:41.688 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76293.log 2024-09-14T07:37:41.689 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97044.log: 96.4% -- replaced with /var/log/ceph/cluster2-client.mirror.91166.log.gz 2024-09-14T07:37:41.689 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.97044.log.gz 2024-09-14T07:37:41.689 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35105.log.gz 2024-09-14T07:37:41.689 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81887.log 2024-09-14T07:37:41.690 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66377.log 2024-09-14T07:37:41.690 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81887.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129068.log 2024-09-14T07:37:41.690 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126369.log 2024-09-14T07:37:41.691 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66377.log: /var/log/ceph/cluster1-client.mirror.129068.log: 86.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99247.log 2024-09-14T07:37:41.691 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.129068.log.gz 2024-09-14T07:37:41.693 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126369.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126369.log.gz 2024-09-14T07:37:41.696 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38956.log 2024-09-14T07:37:41.696 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99247.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99247.log.gz 2024-09-14T07:37:41.696 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106593.log 2024-09-14T07:37:41.696 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98879.log 2024-09-14T07:37:41.697 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76293.log.gz 2024-09-14T07:37:41.697 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.106593.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.38956.log.gz 2024-09-14T07:37:41.697 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81887.log.gz 2024-09-14T07:37:41.697 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.106593.log.gz 2024-09-14T07:37:41.698 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88023.log 2024-09-14T07:37:41.698 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57986.log 2024-09-14T07:37:41.698 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.98879.log: /var/log/ceph/cluster2-client.mirror.88023.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88023.log.gz 2024-09-14T07:37:41.698 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.98879.log.gz 2024-09-14T07:37:41.699 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104010.log 2024-09-14T07:37:41.699 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79191.log 2024-09-14T07:37:41.700 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104010.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104010.log.gz 2024-09-14T07:37:41.700 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49877.log 2024-09-14T07:37:41.701 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120391.log 2024-09-14T07:37:41.702 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63483.log 2024-09-14T07:37:41.702 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49877.log: /var/log/ceph/cluster1-client.mirror.120391.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116919.log 2024-09-14T07:37:41.703 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.129509.log.gz 2024-09-14T07:37:41.703 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120129.log 2024-09-14T07:37:41.704 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.116919.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.120391.log.gz 2024-09-14T07:37:41.704 INFO:teuthology.orchestra.run.smithi169.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.116919.log.gz 2024-09-14T07:37:41.704 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105546.log 2024-09-14T07:37:41.705 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50679.log 2024-09-14T07:37:41.705 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66156.log 2024-09-14T07:37:41.706 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105546.log: /var/log/ceph/cluster2-client.mirror.50679.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.105546.log.gz 2024-09-14T07:37:41.706 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39390.log 2024-09-14T07:37:41.706 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133079.log 2024-09-14T07:37:41.707 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66156.log: /var/log/ceph/cluster2-client.mirror.39390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49656.log 2024-09-14T07:37:41.707 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61247.log 2024-09-14T07:37:41.707 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133079.log: /var/log/ceph/cluster2-client.mirror.49656.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.133079.log.gz 2024-09-14T07:37:41.707 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113933.log 2024-09-14T07:37:41.708 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.45724.log 2024-09-14T07:37:41.708 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61247.log: /var/log/ceph/cluster2-client.mirror.113933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97376.log 2024-09-14T07:37:41.709 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.45724.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68391.log.gz 2024-09-14T07:37:41.709 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64076.log 2024-09-14T07:37:41.710 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97376.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57986.log.gz 2024-09-14T07:37:41.710 INFO:teuthology.orchestra.run.smithi169.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.113933.log.gz 2024-09-14T07:37:41.710 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97376.log.gz 2024-09-14T07:37:41.711 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79191.log.gz 2024-09-14T07:37:41.711 INFO:teuthology.orchestra.run.smithi169.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120129.log.gz 2024-09-14T07:37:41.711 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76723.log 2024-09-14T07:37:41.712 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50807.log 2024-09-14T07:37:41.712 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64076.log: /var/log/ceph/cluster2-client.mirror.76723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33426.log 2024-09-14T07:37:41.712 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50807.log: /var/log/ceph/cluster2-client.mirror.33426.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73209.log 2024-09-14T07:37:41.713 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130822.log 2024-09-14T07:37:41.713 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73209.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99614.log 2024-09-14T07:37:41.714 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59758.log 2024-09-14T07:37:41.714 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80372.log 2024-09-14T07:37:41.715 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99614.log: /var/log/ceph/cluster1-client.mirror.59758.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99614.log.gz 2024-09-14T07:37:41.715 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.107234.log.gz 2024-09-14T07:37:41.715 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51458.log 2024-09-14T07:37:41.716 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80372.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49877.log.gz 2024-09-14T07:37:41.716 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.104315.log 2024-09-14T07:37:41.717 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53237.log 2024-09-14T07:37:41.717 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51458.log: /var/log/ceph/cluster2-client.mirror.2.104315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100494.log 2024-09-14T07:37:41.717 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97009.log 2024-09-14T07:37:41.717 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50679.log.gz 2024-09-14T07:37:41.718 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100494.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100494.log.gz 2024-09-14T07:37:41.718 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119561.log 2024-09-14T07:37:41.719 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85662.log 2024-09-14T07:37:41.720 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97009.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50807.log.gz 2024-09-14T07:37:41.720 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.97009.log.gz 2024-09-14T07:37:41.720 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110127.log 2024-09-14T07:37:41.721 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85662.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66156.log.gz 2024-09-14T07:37:41.721 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59758.log.gz 2024-09-14T07:37:41.721 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107846.log 2024-09-14T07:37:41.722 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.110127.log: /var/log/ceph/cluster2-client.mirror.119561.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.119561.log.gz 2024-09-14T07:37:41.722 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110127.log.gz 2024-09-14T07:37:41.723 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120037.log 2024-09-14T07:37:41.724 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107846.log: 94.1% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107846.log.gz 2024-09-14T07:37:41.724 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.1.91867.log.gz 2024-09-14T07:37:41.724 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55940.log 2024-09-14T07:37:41.724 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53237.log.gz 2024-09-14T07:37:41.725 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120037.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120037.log.gz 2024-09-14T07:37:41.725 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63483.log.gz 2024-09-14T07:37:41.725 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61247.log.gz 2024-09-14T07:37:41.726 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85662.log.gz 2024-09-14T07:37:41.726 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80675.log 2024-09-14T07:37:41.727 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82932.log.gz 2024-09-14T07:37:41.727 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8%/var/log/ceph/cluster2-client.mirror.55940.log: -- replaced with /var/log/ceph/cluster2-client.mirror.64076.log.gz 2024-09-14T07:37:41.727 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70456.log 2024-09-14T07:37:41.727 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98246.log 2024-09-14T07:37:41.727 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73209.log.gz 2024-09-14T07:37:41.728 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80675.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.21658.log 2024-09-14T07:37:41.728 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36965.log 2024-09-14T07:37:41.728 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80372.log.gz 2024-09-14T07:37:41.728 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/ceph.tmp-client.admin.21658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119383.log 2024-09-14T07:37:41.729 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98246.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.21658.log.gz 2024-09-14T07:37:41.729 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.98246.log.gz 2024-09-14T07:37:41.729 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127037.log 2024-09-14T07:37:41.729 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36256.log 2024-09-14T07:37:41.730 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119383.log: /var/log/ceph/cluster2-client.mirror.127037.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127037.log.gz 2024-09-14T07:37:41.731 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38270.log 2024-09-14T07:37:41.731 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106638.log 2024-09-14T07:37:41.731 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123730.log 2024-09-14T07:37:41.732 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.106638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39014.log 2024-09-14T07:37:41.733 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123730.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66377.log.gz 2024-09-14T07:37:41.733 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94136.log.gz 2024-09-14T07:37:41.733 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123730.log.gz 2024-09-14T07:37:41.733 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.106638.log.gz 2024-09-14T07:37:41.734 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125757.log 2024-09-14T07:37:41.734 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68454.log 2024-09-14T07:37:41.735 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39014.log: /var/log/ceph/cluster2-client.mirror.125757.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78449.log 2024-09-14T07:37:41.735 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125757.log.gzgzip 2024-09-14T07:37:41.735 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52574.log 2024-09-14T07:37:41.736 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78449.log: /var/log/ceph/cluster1-client.mirror.68454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98378.log 2024-09-14T07:37:41.736 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58207.log 2024-09-14T07:37:41.737 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98378.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.98378.log.gz 2024-09-14T07:37:41.737 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.78079.log 2024-09-14T07:37:41.738 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110446.log 2024-09-14T07:37:41.738 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46339.log 2024-09-14T07:37:41.738 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.78079.log: /var/log/ceph/cluster2-client.mirror.110446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125706.log 2024-09-14T07:37:41.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110446.log.gz 2024-09-14T07:37:41.740 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46339.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134301.log 2024-09-14T07:37:41.741 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125706.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39049.log 2024-09-14T07:37:41.741 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68539.log 2024-09-14T07:37:41.741 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39049.log: /var/log/ceph/cluster2-client.admin.134301.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134301.log.gz 2024-09-14T07:37:41.742 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125706.log.gz 2024-09-14T07:37:41.742 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49656.log.gz 2024-09-14T07:37:41.744 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51458.log.gz 2024-09-14T07:37:41.745 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.45724.log.gz 2024-09-14T07:37:41.746 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134431.log 2024-09-14T07:37:41.746 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.68539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71723.log 2024-09-14T07:37:41.747 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79999.log 2024-09-14T07:37:41.747 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134431.log: /var/log/ceph/cluster2-client.mirror.71723.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134431.log.gz 2024-09-14T07:37:41.747 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38270.log.gz 2024-09-14T07:37:41.747 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56219.log 2024-09-14T07:37:41.748 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65726.log 2024-09-14T07:37:41.748 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.39049.log.gz 2024-09-14T07:37:41.748 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39014.log.gz 2024-09-14T07:37:41.748 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56219.log: /var/log/ceph/cluster2-client.mirror.79999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107981.log 2024-09-14T07:37:41.749 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.46339.log.gz 2024-09-14T07:37:41.749 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30811.log 2024-09-14T07:37:41.750 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36907.log 2024-09-14T07:37:41.750 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107981.log: /var/log/ceph/cluster1-client.mirror.30811.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107981.log.gz 2024-09-14T07:37:41.751 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- 94.3% /var/log/ceph/cluster2-client.mirror.50865.log 2024-09-14T07:37:41.751 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.104315.log.gz 2024-09-14T07:37:41.751 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125540.log 2024-09-14T07:37:41.751 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36907.log: /var/log/ceph/cluster2-client.mirror.50865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63018.log 2024-09-14T07:37:41.752 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125540.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127920.log 2024-09-14T07:37:41.752 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63018.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125540.log.gz 2024-09-14T07:37:41.754 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52574.log.gz 2024-09-14T07:37:41.754 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38549.log 2024-09-14T07:37:41.754 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83705.log 2024-09-14T07:37:41.754 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127920.log.gz 2024-09-14T07:37:41.754 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133141.log 2024-09-14T07:37:41.756 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83705.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80675.log.gz 2024-09-14T07:37:41.756 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.36256.log.gz 2024-09-14T07:37:41.756 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85207.log 2024-09-14T07:37:41.756 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36907.log.gz 2024-09-14T07:37:41.756 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36965.log.gz 2024-09-14T07:37:41.756 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37991.log 2024-09-14T07:37:41.757 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58207.log.gz 2024-09-14T07:37:41.757 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83235.log 2024-09-14T07:37:41.757 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133141.log: /var/log/ceph/cluster2-client.mirror.37991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85164.log 2024-09-14T07:37:41.757 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46432.log 2024-09-14T07:37:41.757 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83235.log: /var/log/ceph/cluster1-client.mirror.85164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50307.log 2024-09-14T07:37:41.757 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79560.log 2024-09-14T07:37:41.758 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50307.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133141.log.gz 2024-09-14T07:37:41.758 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91409.log.gz 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108409.log 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101681.log 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65006.log 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.101681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85914.log 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130774.log 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.101681.log.gz 2024-09-14T07:37:41.760 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74267.log 2024-09-14T07:37:41.761 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45109.log 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74267.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127493.log 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.130774.log.gz 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45109.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72688.log 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31084.log 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127493.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127493.log.gz 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77595.log 2024-09-14T07:37:41.762 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94795.log 2024-09-14T07:37:41.763 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31084.log: /var/log/ceph/cluster2-client.mirror.77595.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65176.log.gz 2024-09-14T07:37:41.763 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72688.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108409.log.gz 2024-09-14T07:37:41.763 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70456.log.gz 2024-09-14T07:37:41.764 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60410.log 2024-09-14T07:37:41.764 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.94795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36314.log 2024-09-14T07:37:41.764 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60410.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76723.log.gz 2024-09-14T07:37:41.765 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79560.log.gz 2024-09-14T07:37:41.766 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87353.log 2024-09-14T07:37:41.766 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66470.log 2024-09-14T07:37:41.766 INFO:teuthology.orchestra.run.smithi169.stderr: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94795.log.gz 2024-09-14T07:37:41.766 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128100.log 2024-09-14T07:37:41.766 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87353.log: /var/log/ceph/cluster2-client.mirror.66470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62646.log 2024-09-14T07:37:41.771 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59365.log 2024-09-14T07:37:41.771 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62646.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.128100.log.gz 2024-09-14T07:37:41.771 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64820.log 2024-09-14T07:37:41.772 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49094.log.gz 2024-09-14T07:37:41.772 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131029.log 2024-09-14T07:37:41.772 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133312.log 2024-09-14T07:37:41.772 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87626.log 2024-09-14T07:37:41.772 INFO:teuthology.orchestra.run.smithi169.stderr: 83.3% -- replaced with /var/log/ceph/cluster1-client.mirror.131029.log.gz 2024-09-14T07:37:41.772 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110082.log 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61902.log 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87626.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87626.log.gz 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.110082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38305.log 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61902.log: 91.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96772.log 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.110082.log.gz 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.133312.log.gz 2024-09-14T07:37:41.773 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38305.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98095.log 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47734.log 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50307.log.gz 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51272.log 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98095.log.gz 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72595.log 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52074.log 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51272.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50865.log.gz 2024-09-14T07:37:41.774 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72595.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65726.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117951.log 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45417.log 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52074.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59365.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101516.log 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61902.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115677.log 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117951.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101516.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71723.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67836.log 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81114.log 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101516.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47734.log.gz 2024-09-14T07:37:41.778 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45417.log: /var/log/ceph/cluster2-client.mirror.115677.log: /var/log/ceph/cluster2-client.mirror.67836.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115677.log.gz 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88883.log 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126724.log 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81114.log: /var/log/ceph/cluster2-client.mirror.88883.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53907.log 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88883.log.gz 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125489.log 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126724.log: /var/log/ceph/cluster1-client.admin.53907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33333.log 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60410.log.gz 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125489.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126724.log.gz 2024-09-14T07:37:41.779 INFO:teuthology.orchestra.run.smithi169.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125489.log.gz 2024-09-14T07:37:41.780 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95836.log 2024-09-14T07:37:41.780 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66906.log 2024-09-14T07:37:41.781 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33333.log: /var/log/ceph/cluster2-client.mirror.95836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126215.log 2024-09-14T07:37:41.781 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95836.log.gz 2024-09-14T07:37:41.781 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66906.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88187.log 2024-09-14T07:37:41.782 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30180.log 2024-09-14T07:37:41.782 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126215.log: /var/log/ceph/cluster2-client.mirror.88187.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37372.log 2024-09-14T07:37:41.782 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37840.log 2024-09-14T07:37:41.782 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30180.log.gz 2024-09-14T07:37:41.783 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37372.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126215.log.gz 2024-09-14T07:37:41.783 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68773.log 2024-09-14T07:37:41.784 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121967.log 2024-09-14T07:37:41.784 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123616.log 2024-09-14T07:37:41.784 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52776.log 2024-09-14T07:37:41.785 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123616.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121967.log.gz 2024-09-14T07:37:41.785 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123616.log.gz 2024-09-14T07:37:41.786 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129203.log 2024-09-14T07:37:41.786 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.52776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35605.log 2024-09-14T07:37:41.786 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122202.log 2024-09-14T07:37:41.786 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129203.log.gz 2024-09-14T07:37:41.787 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.130936.log 2024-09-14T07:37:41.787 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122202.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105114.log 2024-09-14T07:37:41.787 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.130936.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122202.log.gz 2024-09-14T07:37:41.789 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83705.log.gz 2024-09-14T07:37:41.789 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37991.log.gz 2024-09-14T07:37:41.789 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38305.log.gz 2024-09-14T07:37:41.790 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.38549.log.gz 2024-09-14T07:37:41.790 INFO:teuthology.orchestra.run.smithi169.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30811.log.gz 2024-09-14T07:37:41.790 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46432.log.gz 2024-09-14T07:37:41.790 INFO:teuthology.orchestra.run.smithi169.stderr: 57.1% -- replaced with /var/log/ceph/cluster2-client.mirror.31084.log.gz 2024-09-14T07:37:41.790 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52074.log.gz 2024-09-14T07:37:41.792 INFO:teuthology.orchestra.run.smithi169.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.0.130936.log.gz 2024-09-14T07:37:41.792 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83982.log 2024-09-14T07:37:41.794 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105114.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116568.log 2024-09-14T07:37:41.794 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90087.log 2024-09-14T07:37:41.795 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48536.log 2024-09-14T07:37:41.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116568.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72688.log.gz 2024-09-14T07:37:41.796 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116568.log.gz 2024-09-14T07:37:41.796 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90087.log: 84.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130294.log 2024-09-14T07:37:41.797 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.90087.log.gz 2024-09-14T07:37:41.797 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105015.log 2024-09-14T07:37:41.798 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81114.log.gz 2024-09-14T07:37:41.798 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50063.log 2024-09-14T07:37:41.798 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130294.log.gz 2024-09-14T07:37:41.798 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69062.log 2024-09-14T07:37:41.798 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50063.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105015.log.gz 2024-09-14T07:37:41.798 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.117059.log 2024-09-14T07:37:41.799 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78449.log.gz 2024-09-14T07:37:41.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69062.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117835.log 2024-09-14T07:37:41.799 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.36314.log.gz 2024-09-14T07:37:41.799 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116815.log 2024-09-14T07:37:41.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.117059.log: /var/log/ceph/cluster2-client.mirror.117835.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117835.log.gz 2024-09-14T07:37:41.805 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79999.log.gz 2024-09-14T07:37:41.805 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37372.log.gz 2024-09-14T07:37:41.806 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45109.log.gz 2024-09-14T07:37:41.810 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62646.log.gz 2024-09-14T07:37:41.810 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63018.log.gz 2024-09-14T07:37:41.812 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34581.log 2024-09-14T07:37:41.812 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48536.log.gz 2024-09-14T07:37:41.813 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.116815.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66470.log.gz 2024-09-14T07:37:41.814 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66906.log.gz 2024-09-14T07:37:41.815 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.116815.log.gz 2024-09-14T07:37:41.815 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110359.log 2024-09-14T07:37:41.816 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34581.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83982.log.gz 2024-09-14T07:37:41.816 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115115.log 2024-09-14T07:37:41.817 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110359.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110359.log.gz 2024-09-14T07:37:41.817 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77595.log.gz 2024-09-14T07:37:41.818 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39762.log 2024-09-14T07:37:41.818 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115115.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.115115.log.gz 2024-09-14T07:37:41.819 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34174.log 2024-09-14T07:37:41.819 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57184.log 2024-09-14T07:37:41.820 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102885.log 2024-09-14T07:37:41.821 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44584.log 2024-09-14T07:37:41.822 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102885.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102885.log.gz 2024-09-14T07:37:41.822 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94114.log 2024-09-14T07:37:41.823 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43247.log 2024-09-14T07:37:41.825 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.94114.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94114.log.gz 2024-09-14T07:37:41.826 INFO:teuthology.orchestra.run.smithi169.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87353.log.gz 2024-09-14T07:37:41.830 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87376.log 2024-09-14T07:37:41.831 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121348.log 2024-09-14T07:37:41.834 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87376.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.117059.log.gz 2024-09-14T07:37:41.834 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112521.log 2024-09-14T07:37:41.835 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68598.log 2024-09-14T07:37:41.836 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121348.log: 88.0%/var/log/ceph/cluster2-client.mirror.112521.log: -- replaced with /var/log/ceph/cluster2-client.mirror.121348.log.gz 2024-09-14T07:37:41.836 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112521.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54394.log 2024-09-14T07:37:41.836 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.836 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84268.log 2024-09-14T07:37:41.837 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54394.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62960.log 2024-09-14T07:37:41.838 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93267.log 2024-09-14T07:37:41.838 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84268.log: /var/log/ceph/cluster2-client.mirror.62960.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78383.log 2024-09-14T07:37:41.838 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33930.log 2024-09-14T07:37:41.838 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93267.log: /var/log/ceph/cluster2-client.mirror.78383.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56719.log 2024-09-14T07:37:41.839 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.93267.log.gz 2024-09-14T07:37:41.839 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42689.log 2024-09-14T07:37:41.839 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114556.log 2024-09-14T07:37:41.840 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.30405.log 2024-09-14T07:37:41.840 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42689.log: /var/log/ceph/cluster2-client.mirror.114556.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114556.log.gz 2024-09-14T07:37:41.841 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127515.log 2024-09-14T07:37:41.844 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.30405.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30719.log 2024-09-14T07:37:41.845 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123383.log 2024-09-14T07:37:41.845 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127515.log: /var/log/ceph/cluster1-client.admin.30719.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30719.log.gz 2024-09-14T07:37:41.845 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127515.log.gz 2024-09-14T07:37:41.845 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71107.log 2024-09-14T07:37:41.846 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123383.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128505.log 2024-09-14T07:37:41.846 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.123383.log.gz 2024-09-14T07:37:41.846 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61154.log 2024-09-14T07:37:41.847 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119739.log 2024-09-14T07:37:41.847 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128505.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.128505.log.gz 2024-09-14T07:37:41.851 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61154.log: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.130822.log.gz 2024-09-14T07:37:41.851 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.30505.log.gz 2024-09-14T07:37:41.851 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52353.log 2024-09-14T07:37:41.852 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63669.log 2024-09-14T07:37:41.852 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119739.log: 88.1%/var/log/ceph/cluster2-client.mirror.52353.log: -- replaced with /var/log/ceph/cluster2-client.mirror.119739.log.gz 2024-09-14T07:37:41.853 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107100.log 2024-09-14T07:37:41.853 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63669.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124928.log 2024-09-14T07:37:41.853 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85250.log 2024-09-14T07:37:41.855 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107100.log: /var/log/ceph/cluster2-client.mirror.124928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.31834.log 2024-09-14T07:37:41.855 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107100.log.gz 2024-09-14T07:37:41.855 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.124928.log.gz/var/log/ceph/cluster1-client.mirror.85250.log: 2024-09-14T07:37:41.855 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120072.log 2024-09-14T07:37:41.855 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.3.31834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34954.log 2024-09-14T07:37:41.855 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33147.log 2024-09-14T07:37:41.856 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34954.log: /var/log/ceph/cluster2-client.mirror.120072.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120072.log.gz 2024-09-14T07:37:41.856 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31924.log 2024-09-14T07:37:41.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58300.log 2024-09-14T07:37:41.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.31924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44398.log 2024-09-14T07:37:41.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58300.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113876.log 2024-09-14T07:37:41.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44398.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101566.log 2024-09-14T07:37:41.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113876.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128685.log 2024-09-14T07:37:41.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101566.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.113876.log.gz 2024-09-14T07:37:41.859 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101566.log.gz 2024-09-14T07:37:41.859 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40347.log 2024-09-14T07:37:41.859 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127763.log 2024-09-14T07:37:41.859 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128685.log.gz 2024-09-14T07:37:41.860 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86343.log 2024-09-14T07:37:41.860 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117983.log 2024-09-14T07:37:41.861 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83235.log.gz 2024-09-14T07:37:41.861 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127763.log.gz 2024-09-14T07:37:41.861 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58451.log 2024-09-14T07:37:41.861 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.117983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75135.log 2024-09-14T07:37:41.862 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58451.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.117983.log.gz 2024-09-14T07:37:41.862 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90409.log 2024-09-14T07:37:41.862 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.75135.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61995.log 2024-09-14T07:37:41.863 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124306.log 2024-09-14T07:37:41.863 INFO:teuthology.orchestra.run.smithi169.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90409.log.gz 2024-09-14T07:37:41.863 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120314.log 2024-09-14T07:37:41.863 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124306.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130796.log 2024-09-14T07:37:41.864 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120314.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.120314.log.gz 2024-09-14T07:37:41.864 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124306.log.gz 2024-09-14T07:37:41.865 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104516.log 2024-09-14T07:37:41.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68411.log 2024-09-14T07:37:41.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104516.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130796.log.gz 2024-09-14T07:37:41.866 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.104516.log.gz 2024-09-14T07:37:41.866 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69197.log 2024-09-14T07:37:41.866 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.68411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41418.log 2024-09-14T07:37:41.866 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38735.log 2024-09-14T07:37:41.867 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41418.log: /var/log/ceph/cluster2-client.mirror.69197.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33426.log.gz 2024-09-14T07:37:41.868 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54394.log.gz 2024-09-14T07:37:41.868 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56219.log.gz 2024-09-14T07:37:41.868 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68773.log.gz 2024-09-14T07:37:41.869 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55940.log.gz 2024-09-14T07:37:41.869 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50063.log.gz 2024-09-14T07:37:41.869 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51272.log.gz 2024-09-14T07:37:41.869 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61995.log.gz 2024-09-14T07:37:41.871 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.admin.68539.log.gz 2024-09-14T07:37:41.872 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71107.log.gz 2024-09-14T07:37:41.872 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56033.log 2024-09-14T07:37:41.872 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62960.log.gz 2024-09-14T07:37:41.872 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72595.log.gz 2024-09-14T07:37:41.874 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38119.log 2024-09-14T07:37:41.875 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29740.log 2024-09-14T07:37:41.875 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69956.log 2024-09-14T07:37:41.875 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125828.log 2024-09-14T07:37:41.875 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29740.log: /var/log/ceph/cluster2-client.mirror.69956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129135.log 2024-09-14T07:37:41.876 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121425.log 2024-09-14T07:37:41.876 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129135.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129135.log.gz 2024-09-14T07:37:41.877 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125828.log.gz 2024-09-14T07:37:41.880 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128415.log 2024-09-14T07:37:41.881 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.96183.log 2024-09-14T07:37:41.881 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128415.log: 87.9%gzip -5 -- replaced with /var/log/ceph/cluster2-client.mirror.121425.log.gz --verbose 2024-09-14T07:37:41.881 INFO:teuthology.orchestra.run.smithi169.stderr: -- /var/log/ceph/cluster2-client.mirror.51795.log 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128415.log.gz 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130396.log 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.96183.log: /var/log/ceph/cluster2-client.mirror.51795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40023.log 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130396.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56626.log 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40023.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.130396.log.gz 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112162.log 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56626.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33930.log.gz 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80201.log 2024-09-14T07:37:41.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112162.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52728.log 2024-09-14T07:37:41.886 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80201.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112162.log.gz 2024-09-14T07:37:41.886 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.34174.log.gz 2024-09-14T07:37:41.886 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33744.log 2024-09-14T07:37:41.886 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.52728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97804.log 2024-09-14T07:37:41.886 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132898.log 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97804.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120659.log 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132898.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97804.log.gz 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.132898.log.gz 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.34581.log.gz 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107620.log 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108715.log 2024-09-14T07:37:41.888 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107620.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.120659.log.gz 2024-09-14T07:37:41.889 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107620.log.gz 2024-09-14T07:37:41.889 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35605.log.gz 2024-09-14T07:37:41.889 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107407.log 2024-09-14T07:37:41.889 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108715.log.gz 2024-09-14T07:37:41.890 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84106.log 2024-09-14T07:37:41.890 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94002.log 2024-09-14T07:37:41.890 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84106.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107407.log.gz 2024-09-14T07:37:41.891 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.45819.log 2024-09-14T07:37:41.891 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94002.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37840.log.gz 2024-09-14T07:37:41.891 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.1.45819.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94002.log.gz 2024-09-14T07:37:41.891 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112367.log 2024-09-14T07:37:41.892 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125899.log 2024-09-14T07:37:41.892 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112367.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86627.log 2024-09-14T07:37:41.893 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38119.log.gz 2024-09-14T07:37:41.893 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38735.log.gz 2024-09-14T07:37:41.893 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125899.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112367.log.gz 2024-09-14T07:37:41.893 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125899.log.gz 2024-09-14T07:37:41.893 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.53907.log.gz 2024-09-14T07:37:41.895 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52353.log.gz 2024-09-14T07:37:41.896 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86627.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69062.log.gz 2024-09-14T07:37:41.897 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97672.log 2024-09-14T07:37:41.897 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.31924.log.gz 2024-09-14T07:37:41.897 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41418.log.gz 2024-09-14T07:37:41.897 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43247.log.gz 2024-09-14T07:37:41.898 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44584.log.gz 2024-09-14T07:37:41.900 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130261.log 2024-09-14T07:37:41.901 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79863.log 2024-09-14T07:37:41.901 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78383.log.gz 2024-09-14T07:37:41.902 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97672.log: /var/log/ceph/cluster1-client.mirror.130261.log: 88.0% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.130261.log.gz 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.97672.log.gz 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56905.log 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.admin.30760.log.gz 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42410.log 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37619.log 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56905.log: /var/log/ceph/cluster2-client.mirror.42410.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56719.log.gz 2024-09-14T07:37:41.903 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134327.log 2024-09-14T07:37:41.905 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108545.log 2024-09-14T07:37:41.905 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30743.log 2024-09-14T07:37:41.905 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134327.log: /var/log/ceph/cluster2-client.mirror.108545.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134327.log.gz 2024-09-14T07:37:41.905 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108545.log.gzgzip 2024-09-14T07:37:41.906 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120371.log 2024-09-14T07:37:41.906 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53083.log 2024-09-14T07:37:41.906 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30743.log: /var/log/ceph/cluster2-client.mirror.120371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124735.log 2024-09-14T07:37:41.907 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53083.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2024-09-14T07:37:41.907 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124735.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.120371.log.gz 2024-09-14T07:37:41.907 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111689.log 2024-09-14T07:37:41.907 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75015.log 2024-09-14T07:37:41.908 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111689.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88187.log.gz 2024-09-14T07:37:41.909 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111689.log.gz 2024-09-14T07:37:41.909 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124735.log.gz 2024-09-14T07:37:41.909 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74046.log 2024-09-14T07:37:41.914 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105384.log 2024-09-14T07:37:41.914 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74046.log: /var/log/ceph/cluster2-client.mirror.75015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36872.log 2024-09-14T07:37:41.914 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39797.log 2024-09-14T07:37:41.914 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.105384.log.gz 2024-09-14T07:37:41.914 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132274.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36872.log: /var/log/ceph/cluster2-client.mirror.39797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83402.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63204.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132274.log: /var/log/ceph/cluster2-client.mirror.83402.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124081.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61154.log.gz 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63204.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57184.log.gz 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.132274.log.gz 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr: 79.9% -- replaced with /var/log/ceph/cluster1-client.admin.30743.log.gz 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34116.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102199.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32124.log 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69956.log.gz 2024-09-14T07:37:41.915 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.124081.log.gz 2024-09-14T07:37:41.916 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98060.log 2024-09-14T07:37:41.916 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102199.log.gz 2024-09-14T07:37:41.916 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32124.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77502.log 2024-09-14T07:37:41.916 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58300.log.gz 2024-09-14T07:37:41.916 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84106.log.gz 2024-09-14T07:37:41.916 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124519.log 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98060.log.gz 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86627.log.gz 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80201.log.gz 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85379.log 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77502.log: /var/log/ceph/cluster2-client.mirror.124519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49377.log 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54766.log 2024-09-14T07:37:41.935 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124519.log.gz 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49377.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127673.log 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71444.log 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95281.log 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127673.log.gz 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123659.log 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.96183.log.gz 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45417.log.gz 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127380.log 2024-09-14T07:37:41.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95281.log: /var/log/ceph/cluster2-client.mirror.123659.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95281.log.gz 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128910.log 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126073.log 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127380.log: 86.7%/var/log/ceph/cluster1-client.mirror.128910.log: -- replaced with /var/log/ceph/cluster1-client.mirror.127380.log.gz 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53048.log 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.123659.log.gz 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128910.log.gz 2024-09-14T07:37:41.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126073.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.30480.log 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75789.log 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.3.30480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95358.log 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85914.log.gz 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36163.log 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95358.log: /var/log/ceph/cluster2-client.mirror.36163.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70828.log 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126073.log.gz 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95358.log.gz 2024-09-14T07:37:41.938 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64541.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70979.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114171.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64541.log: /var/log/ceph/cluster2-client.mirror.70979.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127134.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86387.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127134.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114780.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114171.log: 1.3% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.105114.log.gz 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.127134.log.gz/var/log/ceph/cluster2-client.mirror.86387.log: 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114171.log.gz 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101210.log 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.114780.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.114780.log.gz 2024-09-14T07:37:41.939 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84945.log 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101210.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56405.log 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84945.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101210.log.gz 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72932.log 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42724.log 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114945.log 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73023.log 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114945.log: 87.7% -- replaced with /var/log/ceph/cluster2.log.gz 2024-09-14T07:37:41.940 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100140.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72932.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114945.log.gz 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126937.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73023.log: /var/log/ceph/cluster2-client.mirror.100140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85704.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100140.log.gz 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96868.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126937.log: /var/log/ceph/cluster2-client.mirror.85704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45186.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96868.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72037.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70084.log 2024-09-14T07:37:41.941 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96868.log.gz 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89654.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87770.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70084.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126937.log.gz 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70049.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104457.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.89654.log.gz/var/log/ceph/cluster2-client.mirror.87770.log: 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70049.log: 84.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.87770.log.gz 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126440.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79393.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104457.log: /var/log/ceph/cluster2-client.mirror.126440.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57800.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104457.log.gz 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79393.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127718.log 2024-09-14T07:37:41.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57800.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78079.log 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83740.log 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127718.log.gz 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93645.log 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126440.log.gz/var/log/ceph/cluster2-client.mirror.83740.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33802.log 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44491.log 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93645.log: /var/log/ceph/cluster2-client.mirror.33802.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127290.log 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93645.log.gz 2024-09-14T07:37:41.943 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124877.log 2024-09-14T07:37:41.944 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44491.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.83875.log 2024-09-14T07:37:41.944 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127290.log: /var/log/ceph/cluster2-client.mirror.124877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86547.log 2024-09-14T07:37:41.944 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127290.log.gz 2024-09-14T07:37:41.944 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49470.log 2024-09-14T07:37:41.944 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.1.83875.log: /var/log/ceph/cluster2-client.mirror.86547.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124877.log.gz 2024-09-14T07:37:41.952 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.59438.log 2024-09-14T07:37:41.952 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52907.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.2.59438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132024.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115793.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50342.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115793.log: gzip 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132024.log.gz -5 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.58672.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115793.log.gz 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50342.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115528.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67592.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58672.log: /var/log/ceph/cluster2-client.mirror.115528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109099.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70607.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.109099.log: --verbose -- /var/log/ceph/cluster2-client.mirror.41980.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102477.log 2024-09-14T07:37:41.953 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109099.log.gz 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70607.log: /var/log/ceph/cluster2-client.mirror.41980.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88776.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58079.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102477.log.gz 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120609.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58079.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98127.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35233.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120609.log: /var/log/ceph/cluster1-client.mirror.98127.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.120609.log.gz 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.98127.log.gz 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110228.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37898.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110228.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110228.log.gz 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55475.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53715.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67371.log 2024-09-14T07:37:41.954 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84882.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53715.log: /var/log/ceph/cluster2-client.mirror.67371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38677.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92686.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84882.log: /var/log/ceph/cluster2-client.mirror.38677.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.92254.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53650.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.92686.log: /var/log/ceph/cluster1-client.mirror.2.92254.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92686.log.gz 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119383.log.gz 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50900.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.53650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132858.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106239.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132858.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132858.log.gz 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71351.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.106239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109469.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71351.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77002.log 2024-09-14T07:37:41.955 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33333.log.gz 2024-09-14T07:37:41.956 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109469.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56033.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56405.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.106239.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79393.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49377.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49470.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79863.log.gz 2024-09-14T07:37:41.957 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34488.log 2024-09-14T07:37:41.961 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77002.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71444.log.gz 2024-09-14T07:37:41.961 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53083.log.gz 2024-09-14T07:37:41.961 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53048.log.gz 2024-09-14T07:37:41.962 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54766.log.gz 2024-09-14T07:37:41.962 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63669.log.gz 2024-09-14T07:37:41.969 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67836.log.gz 2024-09-14T07:37:41.973 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29818.log 2024-09-14T07:37:41.973 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67685.log 2024-09-14T07:37:41.973 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25405.log 2024-09-14T07:37:41.973 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29818.log.gz 2024-09-14T07:37:41.981 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90022.log 2024-09-14T07:37:41.981 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.25405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57370.log 2024-09-14T07:37:41.981 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25405.log.gz 2024-09-14T07:37:41.981 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69584.log 2024-09-14T07:37:41.984 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53876.log 2024-09-14T07:37:41.989 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117754.log 2024-09-14T07:37:41.990 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.53876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73709.log 2024-09-14T07:37:41.991 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117754.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117754.log.gz 2024-09-14T07:37:41.991 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.67927.log 2024-09-14T07:37:41.992 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97275.log 2024-09-14T07:37:41.993 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.3.67927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37465.log 2024-09-14T07:37:41.994 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.97275.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39390.log.gz 2024-09-14T07:37:41.995 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39762.log.gz 2024-09-14T07:37:41.995 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.97275.log.gz 2024-09-14T07:37:41.995 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119877.log 2024-09-14T07:37:41.996 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40023.log.gz 2024-09-14T07:37:41.996 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30965.log 2024-09-14T07:37:41.996 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32124.log.gz 2024-09-14T07:37:41.997 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119877.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34116.log.gz 2024-09-14T07:37:41.998 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.119877.log.gz 2024-09-14T07:37:41.998 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34488.log.gz 2024-09-14T07:37:41.999 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126562.log 2024-09-14T07:37:41.999 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30965.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.34954.log.gz 2024-09-14T07:37:41.999 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86320.log 2024-09-14T07:37:42.000 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126562.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126562.log.gz 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35233.log.gz 2024-09-14T07:37:42.000 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.000 INFO:teuthology.orchestra.run.smithi169.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.37465.log.gz 2024-09-14T07:37:42.001 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37619.log.gz 2024-09-14T07:37:42.001 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40347.log.gz 2024-09-14T07:37:42.001 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67278.log 2024-09-14T07:37:42.002 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86320.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29792.log 2024-09-14T07:37:42.002 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87229.log 2024-09-14T07:37:42.003 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29792.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50249.log 2024-09-14T07:37:42.004 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128843.log 2024-09-14T07:37:42.004 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88492.log.gz 2024-09-14T07:37:42.004 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50249.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36872.log.gz 2024-09-14T07:37:42.005 INFO:teuthology.orchestra.run.smithi169.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.92254.log.gz 2024-09-14T07:37:42.005 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133287.log 2024-09-14T07:37:42.006 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128843.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122658.log 2024-09-14T07:37:42.006 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128843.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46153.log 2024-09-14T07:37:42.006 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133287.log: /var/log/ceph/cluster2-client.mirror.122658.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44398.log.gz 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.133287.log.gz 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122658.log.gz 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59614.log 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65064.log 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32717.log 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29740.log.gz 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29792.log.gz 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86839.log 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65064.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44491.log.gz 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32717.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74604.log 2024-09-14T07:37:42.010 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58486.log 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74604.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64041.log 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86839.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56905.log.gz 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58486.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110642.log 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95631.log 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129973.log 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95631.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110642.log.gz 2024-09-14T07:37:42.011 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75452.log 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89784.log 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95631.log.gz 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129973.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74232.log.gz 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75452.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74267.log.gz 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124977.log 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89784.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100951.log 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.129973.log.gz 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124977.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89784.log.gz 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124977.log.gz 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.014 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57370.log.gz 2024-09-14T07:37:42.015 INFO:teuthology.orchestra.run.smithi169.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.30965.log.gz 2024-09-14T07:37:42.015 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70607.log.gz 2024-09-14T07:37:42.015 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33802.log.gz 2024-09-14T07:37:42.015 INFO:teuthology.orchestra.run.smithi169.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85704.log.gz 2024-09-14T07:37:42.015 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84684.log 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100951.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64541.log.gz 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125352.log 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84684.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75452.log.gz 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67278.log.gz 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100951.log.gz 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77502.log.gz 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74046.log.gz 2024-09-14T07:37:42.016 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100035.log 2024-09-14T07:37:42.017 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125352.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125352.log.gz 2024-09-14T07:37:42.017 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75015.log.gz 2024-09-14T07:37:42.017 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53514.log 2024-09-14T07:37:42.018 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100035.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.100035.log.gz 2024-09-14T07:37:42.018 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31050.log 2024-09-14T07:37:42.018 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115486.log 2024-09-14T07:37:42.019 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67685.log.gz 2024-09-14T07:37:42.019 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31050.log: /var/log/ceph/cluster1-client.mirror.53514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125048.log 2024-09-14T07:37:42.019 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115486.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107652.log 2024-09-14T07:37:42.019 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74604.log.gz 2024-09-14T07:37:42.020 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112921.log 2024-09-14T07:37:42.021 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.107652.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.125048.log.gz 2024-09-14T07:37:42.022 INFO:teuthology.orchestra.run.smithi169.stderr: 90.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115486.log.gz 2024-09-14T07:37:42.022 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.107652.log.gz 2024-09-14T07:37:42.022 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129442.log 2024-09-14T07:37:42.022 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47885.log 2024-09-14T07:37:42.022 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.129442.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.129442.log.gz 2024-09-14T07:37:42.023 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49412.log 2024-09-14T07:37:42.023 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49319.log 2024-09-14T07:37:42.023 INFO:teuthology.orchestra.run.smithi169.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112921.log.gz 2024-09-14T07:37:42.024 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133515.log 2024-09-14T07:37:42.024 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71851.log 2024-09-14T07:37:42.024 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127695.log 2024-09-14T07:37:42.025 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38491.log 2024-09-14T07:37:42.025 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127695.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127695.log.gz 2024-09-14T07:37:42.025 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127470.log 2024-09-14T07:37:42.026 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38491.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134379.log 2024-09-14T07:37:42.027 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38212.log 2024-09-14T07:37:42.027 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127470.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.96772.log.gz/var/log/ceph/cluster2-client.admin.134379.log: 2024-09-14T07:37:42.027 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134379.log.gz 2024-09-14T07:37:42.027 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127470.log.gz 2024-09-14T07:37:42.027 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62309.log 2024-09-14T07:37:42.028 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133121.log 2024-09-14T07:37:42.028 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75217.log 2024-09-14T07:37:42.028 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62309.log: /var/log/ceph/cluster2-client.mirror.133121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123636.log 2024-09-14T07:37:42.029 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.75217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130142.log 2024-09-14T07:37:42.029 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133121.log.gz 2024-09-14T07:37:42.031 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123636.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.123636.log.gz 2024-09-14T07:37:42.031 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86387.log.gz 2024-09-14T07:37:42.031 INFO:teuthology.orchestra.run.smithi169.stderr: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86320.log.gz 2024-09-14T07:37:42.031 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100363.log 2024-09-14T07:37:42.031 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116036.log 2024-09-14T07:37:42.031 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130142.log: /var/log/ceph/cluster2-client.mirror.100363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98505.log 2024-09-14T07:37:42.032 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130142.log.gz/var/log/ceph/cluster2-client.mirror.116036.log: 2024-09-14T07:37:42.032 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103463.log 2024-09-14T07:37:42.032 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.116036.log.gz 2024-09-14T07:37:42.032 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98505.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98505.log.gz 2024-09-14T07:37:42.033 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114980.log 2024-09-14T07:37:42.033 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69174.log 2024-09-14T07:37:42.034 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103463.log: /var/log/ceph/cluster2-client.mirror.114980.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103463.log.gz 2024-09-14T07:37:42.034 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114980.log.gz 2024-09-14T07:37:42.034 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127650.log 2024-09-14T07:37:42.034 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.69174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109606.log 2024-09-14T07:37:42.035 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87503.log 2024-09-14T07:37:42.036 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127650.log: /var/log/ceph/cluster2-client.mirror.109606.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.127650.log.gz 2024-09-14T07:37:42.036 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109606.log.gz 2024-09-14T07:37:42.036 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99548.log 2024-09-14T07:37:42.037 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129407.log 2024-09-14T07:37:42.037 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87503.log: /var/log/ceph/cluster1-client.mirror.99548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85015.log 2024-09-14T07:37:42.038 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129407.log: 53.5% -- replaced with /var/log/ceph/cluster1-client.mirror.129407.log.gz 2024-09-14T07:37:42.038 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105680.log 2024-09-14T07:37:42.039 INFO:teuthology.orchestra.run.smithi169.stderr: 91.5%/var/log/ceph/cluster2-client.mirror.85015.log: -- replaced with /var/log/ceph/cluster1-client.mirror.99548.log.gz 2024-09-14T07:37:42.039 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51795.log.gz 2024-09-14T07:37:42.039 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102680.log 2024-09-14T07:37:42.039 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128528.log 2024-09-14T07:37:42.040 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105680.log: /var/log/ceph/cluster2-client.mirror.102680.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102680.log.gz 2024-09-14T07:37:42.044 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62925.log 2024-09-14T07:37:42.045 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128528.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.128528.log.gz 2024-09-14T07:37:42.050 INFO:teuthology.orchestra.run.smithi169.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105680.log.gz 2024-09-14T07:37:42.052 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115758.log 2024-09-14T07:37:42.053 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.62925.log: -- replaced with /var/log/ceph/cluster2-client.mirror.33147.log.gz 2024-09-14T07:37:42.053 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120819.log 2024-09-14T07:37:42.054 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115758.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77002.log.gz 2024-09-14T07:37:42.054 INFO:teuthology.orchestra.run.smithi169.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.115758.log.gz 2024-09-14T07:37:42.054 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81821.log 2024-09-14T07:37:42.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32217.log 2024-09-14T07:37:42.055 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120819.log.gz 2024-09-14T07:37:42.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62553.log 2024-09-14T07:37:42.056 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33709.log 2024-09-14T07:37:42.056 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131524.log 2024-09-14T07:37:42.056 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62553.log: /var/log/ceph/cluster2-client.mirror.33709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134353.log 2024-09-14T07:37:42.057 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50900.log.gz 2024-09-14T07:37:42.057 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131524.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72037.log.gz 2024-09-14T07:37:42.059 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131524.log.gz 2024-09-14T07:37:42.059 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47885.log.gz 2024-09-14T07:37:42.059 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62925.log.gz 2024-09-14T07:37:42.059 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66435.log 2024-09-14T07:37:42.060 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134353.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134353.log.gz 2024-09-14T07:37:42.060 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83099.log 2024-09-14T07:37:42.061 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71909.log 2024-09-14T07:37:42.061 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68366.log 2024-09-14T07:37:42.061 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83099.log: /var/log/ceph/cluster2-client.mirror.71909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81316.log 2024-09-14T07:37:42.064 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.68366.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60224.log 2024-09-14T07:37:42.065 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118581.log 2024-09-14T07:37:42.065 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76165.log 2024-09-14T07:37:42.066 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60224.log: /var/log/ceph/cluster2-client.mirror.118581.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118581.log.gz 2024-09-14T07:37:42.066 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85422.log 2024-09-14T07:37:42.067 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85316.log 2024-09-14T07:37:42.067 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71258.log 2024-09-14T07:37:42.068 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85422.log: /var/log/ceph/cluster2-client.mirror.85316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124999.log 2024-09-14T07:37:42.068 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88700.log 2024-09-14T07:37:42.069 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71258.log: /var/log/ceph/cluster2-client.mirror.124999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106660.log 2024-09-14T07:37:42.069 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.124999.log.gz 2024-09-14T07:37:42.069 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55382.log 2024-09-14T07:37:42.079 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106660.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.32717.log.gz 2024-09-14T07:37:42.081 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42503.log 2024-09-14T07:37:42.082 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55382.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83402.log.gz 2024-09-14T07:37:42.083 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38212.log.gz 2024-09-14T07:37:42.084 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46153.log.gz 2024-09-14T07:37:42.088 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.87229.log.gz 2024-09-14T07:37:42.096 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65156.log.gz 2024-09-14T07:37:42.100 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30695.log 2024-09-14T07:37:42.102 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42503.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57800.log.gz 2024-09-14T07:37:42.102 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53514.log.gz 2024-09-14T07:37:42.104 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73023.log.gz 2024-09-14T07:37:42.108 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45144.log 2024-09-14T07:37:42.108 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87503.log.gz 2024-09-14T07:37:42.109 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30695.log: 0.0% 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66435.log.gz 2024-09-14T07:37:42.109 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.30695.log.gz 2024-09-14T07:37:42.110 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70049.log.gz 2024-09-14T07:37:42.111 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.115528.log.gz 2024-09-14T07:37:42.114 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106712.log 2024-09-14T07:37:42.114 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122013.log 2024-09-14T07:37:42.114 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.100363.log.gz 2024-09-14T07:37:42.115 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73709.log.gz 2024-09-14T07:37:42.115 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106712.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106712.log.gz 2024-09-14T07:37:42.115 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133342.log 2024-09-14T07:37:42.115 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25532.log 2024-09-14T07:37:42.115 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122013.log: /var/log/ceph/cluster1-client.mirror.133342.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122013.log.gz 2024-09-14T07:37:42.115 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133342.log.gz 2024-09-14T07:37:42.116 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97931.log 2024-09-14T07:37:42.116 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.75217.log.gz 2024-09-14T07:37:42.117 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59323.log 2024-09-14T07:37:42.117 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.25532.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25532.log.gz 2024-09-14T07:37:42.117 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97931.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.97931.log.gz 2024-09-14T07:37:42.118 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124857.log 2024-09-14T07:37:42.118 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33554.log 2024-09-14T07:37:42.118 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 88.9% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.83099.log.gz 2024-09-14T07:37:42.118 INFO:teuthology.orchestra.run.smithi169.stderr: /var/log/ceph/cluster2-client.mirror.101601.log 2024-09-14T07:37:42.119 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124857.log: /var/log/ceph/cluster2-client.mirror.33554.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44642.log 2024-09-14T07:37:42.119 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.124857.log.gz 2024-09-14T07:37:42.119 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101601.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101601.log.gz 2024-09-14T07:37:42.120 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84684.log.gz 2024-09-14T07:37:42.120 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60538.log 2024-09-14T07:37:42.120 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100175.log 2024-09-14T07:37:42.121 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44642.log: /var/log/ceph/cluster2-client.mirror.60538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123753.log 2024-09-14T07:37:42.121 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30104.log 2024-09-14T07:37:42.122 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100175.log: /var/log/ceph/cluster2-client.mirror.123753.log: 88.2% 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100175.log.gz 2024-09-14T07:37:42.122 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.123753.log.gz 2024-09-14T07:37:42.122 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57463.log 2024-09-14T07:37:42.123 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107019.log 2024-09-14T07:37:42.123 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30104.log.gz 2024-09-14T07:37:42.123 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66720.log 2024-09-14T07:37:42.123 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57463.log: /var/log/ceph/cluster2-client.mirror.107019.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107019.log.gz 2024-09-14T07:37:42.124 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79661.log 2024-09-14T07:37:42.124 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66720.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127358.log 2024-09-14T07:37:42.125 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79661.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119039.log 2024-09-14T07:37:42.125 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29850.log 2024-09-14T07:37:42.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127358.log: /var/log/ceph/cluster2-client.mirror.119039.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127358.log.gz 2024-09-14T07:37:42.126 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.119039.log.gz 2024-09-14T07:37:42.126 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96673.log 2024-09-14T07:37:42.127 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47420.log 2024-09-14T07:37:42.127 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29850.log.gz 2024-09-14T07:37:42.127 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127898.log 2024-09-14T07:37:42.128 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96673.log: /var/log/ceph/cluster2-client.mirror.47420.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87252.log 2024-09-14T07:37:42.128 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.96673.log.gz 2024-09-14T07:37:42.128 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127898.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127898.log.gz 2024-09-14T07:37:42.129 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46897.log 2024-09-14T07:37:42.129 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89595.log 2024-09-14T07:37:42.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126846.log 2024-09-14T07:37:42.134 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89595.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.106660.log.gz 2024-09-14T07:37:42.140 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128550.log 2024-09-14T07:37:42.141 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126846.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126846.log.gz 2024-09-14T07:37:42.141 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.111002.log 2024-09-14T07:37:42.142 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128550.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99050.log 2024-09-14T07:37:42.142 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128550.log.gz/var/log/ceph/cluster1-client.mirror.0.111002.log: 2024-09-14T07:37:42.142 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93359.log 2024-09-14T07:37:42.143 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99050.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99050.log.gz 2024-09-14T07:37:42.144 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40193.log 2024-09-14T07:37:42.146 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93359.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.93359.log.gz 2024-09-14T07:37:42.146 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55010.log 2024-09-14T07:37:42.147 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71293.log 2024-09-14T07:37:42.147 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80271.log 2024-09-14T07:37:42.148 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124906.log 2024-09-14T07:37:42.149 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37186.log 2024-09-14T07:37:42.150 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124906.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124906.log.gz 2024-09-14T07:37:42.151 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58021.log 2024-09-14T07:37:42.151 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45594.log 2024-09-14T07:37:42.154 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58021.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78079.log.gz 2024-09-14T07:37:42.154 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48385.log 2024-09-14T07:37:42.155 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61844.log 2024-09-14T07:37:42.156 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48385.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55847.log 2024-09-14T07:37:42.156 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56068.log 2024-09-14T07:37:42.157 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61751.log 2024-09-14T07:37:42.162 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67650.log 2024-09-14T07:37:42.162 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106844.log 2024-09-14T07:37:42.164 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.68366.log.gz 2024-09-14T07:37:42.165 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49412.log.gz 2024-09-14T07:37:42.165 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58486.log.gz 2024-09-14T07:37:42.165 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58672.log.gz 2024-09-14T07:37:42.165 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59323.log.gz 2024-09-14T07:37:42.166 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71909.log.gz 2024-09-14T07:37:42.167 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42445.log 2024-09-14T07:37:42.168 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106844.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106844.log.gz 2024-09-14T07:37:42.170 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41383.log 2024-09-14T07:37:42.170 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55382.log.gz 2024-09-14T07:37:42.170 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55475.log.gz 2024-09-14T07:37:42.170 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60224.log.gz 2024-09-14T07:37:42.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42445.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31101.log 2024-09-14T07:37:42.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41383.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121544.log 2024-09-14T07:37:42.172 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123701.log 2024-09-14T07:37:42.173 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121544.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121544.log.gz 2024-09-14T07:37:42.173 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47048.log 2024-09-14T07:37:42.176 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123701.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.123701.log.gz 2024-09-14T07:37:42.177 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82493.log 2024-09-14T07:37:42.178 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36628.log 2024-09-14T07:37:42.179 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52388.log 2024-09-14T07:37:42.181 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36628.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85250.log.gz 2024-09-14T07:37:42.182 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71014.log 2024-09-14T07:37:42.183 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74887.log 2024-09-14T07:37:42.184 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71014.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59438.log 2024-09-14T07:37:42.184 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105638.log 2024-09-14T07:37:42.185 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.59438.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83740.log.gz 2024-09-14T07:37:42.185 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.38491.log.gz 2024-09-14T07:37:42.186 INFO:teuthology.orchestra.run.smithi169.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31101.log.gz 2024-09-14T07:37:42.186 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36163.log.gz 2024-09-14T07:37:42.187 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41383.log.gz 2024-09-14T07:37:42.187 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41980.log.gz 2024-09-14T07:37:42.189 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104281.log 2024-09-14T07:37:42.191 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105638.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105638.log.gz 2024-09-14T07:37:42.192 INFO:teuthology.orchestra.run.smithi169.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.111002.log.gz 2024-09-14T07:37:42.192 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56440.log 2024-09-14T07:37:42.192 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104281.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41046.log 2024-09-14T07:37:42.192 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104281.log.gz 2024-09-14T07:37:42.193 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56440.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82089.log 2024-09-14T07:37:42.193 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36407.log 2024-09-14T07:37:42.193 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82089.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67778.log 2024-09-14T07:37:42.194 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87132.log 2024-09-14T07:37:42.194 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36407.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44642.log.gz 2024-09-14T07:37:42.194 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67778.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45144.log.gz 2024-09-14T07:37:42.195 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120508.log 2024-09-14T07:37:42.195 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.87132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72781.log 2024-09-14T07:37:42.195 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120508.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120508.log.gz 2024-09-14T07:37:42.196 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117708.log 2024-09-14T07:37:42.196 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131429.log 2024-09-14T07:37:42.197 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117708.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117708.log.gz 2024-09-14T07:37:42.197 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110856.log 2024-09-14T07:37:42.198 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131429.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.131429.log.gz 2024-09-14T07:37:42.198 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128798.log 2024-09-14T07:37:42.198 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110856.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.110856.log.gz 2024-09-14T07:37:42.199 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128123.log 2024-09-14T07:37:42.199 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128798.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39107.log 2024-09-14T07:37:42.199 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128798.log.gz 2024-09-14T07:37:42.200 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128123.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85893.log 2024-09-14T07:37:42.200 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39107.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128123.log.gz 2024-09-14T07:37:42.200 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93695.log 2024-09-14T07:37:42.201 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85893.log: /var/log/ceph/cluster2-client.mirror.93695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87526.log 2024-09-14T07:37:42.202 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93695.log.gz 2024-09-14T07:37:42.202 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43712.log 2024-09-14T07:37:42.203 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67986.log 2024-09-14T07:37:42.203 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43712.log: /var/log/ceph/cluster2-client.mirror.67986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133427.log 2024-09-14T07:37:42.204 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56626.log.gz 2024-09-14T07:37:42.204 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122277.log 2024-09-14T07:37:42.205 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133427.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58021.log.gz 2024-09-14T07:37:42.205 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58079.log.gz 2024-09-14T07:37:42.205 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85893.log.gz 2024-09-14T07:37:42.205 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67371.log.gz 2024-09-14T07:37:42.206 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66720.log.gz 2024-09-14T07:37:42.207 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133427.log.gz 2024-09-14T07:37:42.209 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81215.log 2024-09-14T07:37:42.209 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48199.log 2024-09-14T07:37:42.209 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81215.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.122277.log.gz 2024-09-14T07:37:42.210 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128595.log 2024-09-14T07:37:42.210 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110734.log 2024-09-14T07:37:42.211 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128595.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128595.log.gz 2024-09-14T07:37:42.211 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55754.log 2024-09-14T07:37:42.211 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110734.log: 95.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88776.log.gz 2024-09-14T07:37:42.212 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31834.log 2024-09-14T07:37:42.212 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76258.log 2024-09-14T07:37:42.212 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.110734.log.gz 2024-09-14T07:37:42.213 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78686.log 2024-09-14T07:37:42.213 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82695.log 2024-09-14T07:37:42.214 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78686.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39483.log 2024-09-14T07:37:42.215 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82695.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83875.log 2024-09-14T07:37:42.215 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62681.log 2024-09-14T07:37:42.216 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39483.log: /var/log/ceph/cluster1-client.mirror.83875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29851.log 2024-09-14T07:37:42.216 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49970.log 2024-09-14T07:37:42.216 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30230.log 2024-09-14T07:37:42.216 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29851.log.gz 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49970.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105613.log 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110050.log 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.105613.log: /var/log/ceph/cluster1-client.admin.30230.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43991.log 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30230.log.gz/var/log/ceph/cluster2-client.mirror.110050.log: 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105613.log.gz 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102079.log 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.110050.log.gz 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37654.log 2024-09-14T07:37:42.218 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43991.log: /var/log/ceph/cluster2-client.mirror.102079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43061.log 2024-09-14T07:37:42.219 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37654.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102079.log.gz 2024-09-14T07:37:42.219 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45572.log 2024-09-14T07:37:42.219 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101396.log 2024-09-14T07:37:42.220 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62181.log 2024-09-14T07:37:42.220 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.45572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52990.log 2024-09-14T07:37:42.220 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101396.log: /var/log/ceph/cluster2-client.mirror.62181.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115615.log 2024-09-14T07:37:42.221 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.101396.log.gz 2024-09-14T07:37:42.222 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75882.log 2024-09-14T07:37:42.222 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75072.log 2024-09-14T07:37:42.222 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115615.log: /var/log/ceph/cluster2-client.mirror.75882.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115615.log.gz 2024-09-14T07:37:42.222 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83437.log 2024-09-14T07:37:42.223 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30008.log 2024-09-14T07:37:42.223 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83437.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127200.log 2024-09-14T07:37:42.223 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106618.log 2024-09-14T07:37:42.223 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30008.log.gz 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69491.log 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127200.log.gz 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64878.log 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106618.log: /var/log/ceph/cluster2-client.mirror.69491.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82629.log 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64878.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83134.log 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109668.log 2024-09-14T07:37:42.225 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83134.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33275.log 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82730.log 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33275.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109668.log.gz 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57277.log 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106618.log.gz 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35477.log 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109571.log 2024-09-14T07:37:42.227 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72223.log 2024-09-14T07:37:42.228 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58916.log 2024-09-14T07:37:42.228 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72223.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.109571.log.gz 2024-09-14T07:37:42.228 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126266.log 2024-09-14T07:37:42.229 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69526.log 2024-09-14T07:37:42.229 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58916.log: /var/log/ceph/cluster2-client.mirror.126266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125777.log 2024-09-14T07:37:42.229 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69526.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126266.log.gz 2024-09-14T07:37:42.232 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122319.log 2024-09-14T07:37:42.232 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68241.log 2024-09-14T07:37:42.233 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100459.log 2024-09-14T07:37:42.234 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125589.log 2024-09-14T07:37:42.234 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100459.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125777.log.gz 2024-09-14T07:37:42.234 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100459.log.gz 2024-09-14T07:37:42.235 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85144.log 2024-09-14T07:37:42.235 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125589.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125589.log.gz 2024-09-14T07:37:42.236 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89024.log 2024-09-14T07:37:42.236 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123974.log 2024-09-14T07:37:42.237 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70270.log 2024-09-14T07:37:42.237 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89024.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89024.log.gz 2024-09-14T07:37:42.238 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123974.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123974.log.gz 2024-09-14T07:37:42.238 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129461.log 2024-09-14T07:37:42.239 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99707.log 2024-09-14T07:37:42.239 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102544.log 2024-09-14T07:37:42.239 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.129461.log: /var/log/ceph/cluster2-client.mirror.99707.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.129461.log.gz 2024-09-14T07:37:42.240 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99707.log.gz 2024-09-14T07:37:42.244 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130444.log 2024-09-14T07:37:42.245 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.102544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101050.log 2024-09-14T07:37:42.245 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.130444.log: -- replaced with /var/log/ceph/cluster1-client.mirror.102544.log.gz 2024-09-14T07:37:42.245 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126102.log 2024-09-14T07:37:42.252 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101050.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86072.log 2024-09-14T07:37:42.258 INFO:teuthology.orchestra.run.smithi169.stderr: 97.1%/var/log/ceph/cluster2-client.mirror.126102.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126102.log.gz 2024-09-14T07:37:42.258 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.101050.log.gz 2024-09-14T07:37:42.258 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89047.log 2024-09-14T07:37:42.259 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.86072.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128978.log 2024-09-14T07:37:42.270 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89047.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.109469.log.gz 2024-09-14T07:37:42.271 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118367.log 2024-09-14T07:37:42.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128978.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.128978.log.gz 2024-09-14T07:37:42.272 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35698.log 2024-09-14T07:37:42.273 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118367.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118367.log.gz 2024-09-14T07:37:42.273 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40613.log 2024-09-14T07:37:42.274 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99503.log 2024-09-14T07:37:42.274 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40613.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112332.log 2024-09-14T07:37:42.275 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.99503.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99503.log.gz 2024-09-14T07:37:42.276 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104934.log 2024-09-14T07:37:42.276 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112332.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.112332.log.gz 2024-09-14T07:37:42.277 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94418.log 2024-09-14T07:37:42.277 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122319.log.gz 2024-09-14T07:37:42.278 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104934.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104934.log.gz 2024-09-14T07:37:42.279 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113504.log 2024-09-14T07:37:42.279 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94418.log: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.90022.log.gz 2024-09-14T07:37:42.279 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123474.log 2024-09-14T07:37:42.279 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94418.log.gz 2024-09-14T07:37:42.279 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113504.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.113504.log.gz 2024-09-14T07:37:42.279 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40270.log 2024-09-14T07:37:42.280 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123474.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133800.log 2024-09-14T07:37:42.280 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105206.log 2024-09-14T07:37:42.281 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133800.log: 82.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133800.log.gz 2024-09-14T07:37:42.282 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133515.log.gz 2024-09-14T07:37:42.282 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86097.log 2024-09-14T07:37:42.283 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39988.log 2024-09-14T07:37:42.283 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105206.log: /var/log/ceph/cluster2-client.mirror.86097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125019.log 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39988.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105206.log.gz 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90656.log 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.123474.log.gz 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60131.log 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90656.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.68620.log 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr: 84.4%/var/log/ceph/cluster2-client.mirror.60131.log: -- replaced with /var/log/ceph/cluster2-client.mirror.90656.log.gz 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103144.log 2024-09-14T07:37:42.285 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31983.log 2024-09-14T07:37:42.286 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.1.68620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133654.log 2024-09-14T07:37:42.287 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103144.log: /var/log/ceph/cluster2-client.mirror.31983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93780.log 2024-09-14T07:37:42.287 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125019.log.gz 2024-09-14T07:37:42.287 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.103144.log.gz 2024-09-14T07:37:42.287 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66621.log 2024-09-14T07:37:42.287 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44863.log 2024-09-14T07:37:42.288 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96638.log 2024-09-14T07:37:42.288 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.93780.log.gz 2024-09-14T07:37:42.288 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82796.log 2024-09-14T07:37:42.288 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44863.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.133654.log.gz 2024-09-14T07:37:42.289 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92254.log 2024-09-14T07:37:42.289 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96638.log.gz 2024-09-14T07:37:42.289 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32066.log 2024-09-14T07:37:42.290 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82796.log: /var/log/ceph/cluster2-client.mirror.92254.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.45819.log.gz 2024-09-14T07:37:42.290 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115461.log 2024-09-14T07:37:42.291 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60596.log 2024-09-14T07:37:42.291 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32066.log: /var/log/ceph/cluster1-client.mirror.115461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34209.log 2024-09-14T07:37:42.291 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.115461.log.gz 2024-09-14T07:37:42.292 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54580.log 2024-09-14T07:37:42.292 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60596.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30777.log 2024-09-14T07:37:42.293 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34209.log: /var/log/ceph/cluster2-client.mirror.54580.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92254.log.gz 2024-09-14T07:37:42.295 INFO:teuthology.orchestra.run.smithi169.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.59438.log.gz 2024-09-14T07:37:42.296 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92751.log 2024-09-14T07:37:42.296 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40825.log 2024-09-14T07:37:42.297 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.92751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125374.log 2024-09-14T07:37:42.297 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112444.log 2024-09-14T07:37:42.297 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70921.log 2024-09-14T07:37:42.298 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112444.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.125374.log.gz 2024-09-14T07:37:42.298 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112444.log.gz 2024-09-14T07:37:42.301 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67650.log.gz 2024-09-14T07:37:42.302 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67778.log.gz 2024-09-14T07:37:42.302 INFO:teuthology.orchestra.run.smithi169.stderr: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88700.log.gz 2024-09-14T07:37:42.302 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67986.log.gz 2024-09-14T07:37:42.303 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53561.log 2024-09-14T07:37:42.303 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.94795.log 2024-09-14T07:37:42.304 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110149.log 2024-09-14T07:37:42.304 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.2.94795.log: 88.8%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.70084.log.gz -5 2024-09-14T07:37:42.304 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.33837.log 2024-09-14T07:37:42.304 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94615.log 2024-09-14T07:37:42.305 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33837.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32996.log 2024-09-14T07:37:42.305 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94615.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94615.log.gz 2024-09-14T07:37:42.306 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114591.log 2024-09-14T07:37:42.306 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130860.log 2024-09-14T07:37:42.307 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114591.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.114591.log.gz 2024-09-14T07:37:42.307 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.30580.log 2024-09-14T07:37:42.308 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130860.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.130860.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.30530.log 2024-09-14T07:37:42.308 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.309 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85187.log 2024-09-14T07:37:42.309 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.3.30580.log: /var/log/ceph/cluster2-client.mirror.1.30530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93334.log 2024-09-14T07:37:42.310 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85187.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107192.log 2024-09-14T07:37:42.310 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.93334.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93334.log.gz 2024-09-14T07:37:42.311 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87959.log 2024-09-14T07:37:42.312 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107192.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107192.log.gz 2024-09-14T07:37:42.313 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33647.log 2024-09-14T07:37:42.319 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87959.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71258.log.gz 2024-09-14T07:37:42.319 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114256.log 2024-09-14T07:37:42.320 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33554.log.gz 2024-09-14T07:37:42.320 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33647.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71851.log.gz 2024-09-14T07:37:42.320 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41046.log.gz 2024-09-14T07:37:42.320 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47048.log.gz 2024-09-14T07:37:42.322 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36407.log.gz 2024-09-14T07:37:42.322 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37186.log.gz 2024-09-14T07:37:42.322 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81316.log.gz 2024-09-14T07:37:42.323 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.53876.log.gz 2024-09-14T07:37:42.323 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37654.log.gz 2024-09-14T07:37:42.323 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.37898.log.gz 2024-09-14T07:37:42.324 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81215.log.gz 2024-09-14T07:37:42.324 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.94795.log.gz 2024-09-14T07:37:42.325 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93730.log 2024-09-14T07:37:42.326 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114256.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.114256.log.gz 2024-09-14T07:37:42.326 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109427.log 2024-09-14T07:37:42.327 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93730.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93730.log.gz 2024-09-14T07:37:42.327 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54580.log.gz 2024-09-14T07:37:42.327 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33709.log.gz 2024-09-14T07:37:42.327 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100629.log 2024-09-14T07:37:42.328 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57463.log.gz 2024-09-14T07:37:42.328 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70921.log.gz 2024-09-14T07:37:42.329 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70979.log.gz 2024-09-14T07:37:42.329 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48385.log.gz 2024-09-14T07:37:42.331 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109427.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64041.log.gz 2024-09-14T07:37:42.332 INFO:teuthology.orchestra.run.smithi169.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.109427.log.gz 2024-09-14T07:37:42.332 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85121.log 2024-09-14T07:37:42.333 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75789.log.gz 2024-09-14T07:37:42.333 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100629.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76258.log.gz 2024-09-14T07:37:42.333 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100629.log.gz 2024-09-14T07:37:42.333 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106408.log 2024-09-14T07:37:42.334 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84291.log 2024-09-14T07:37:42.335 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38584.log 2024-09-14T07:37:42.335 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106408.log: /var/log/ceph/cluster2-client.mirror.84291.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106408.log.gz 2024-09-14T07:37:42.336 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127008.log 2024-09-14T07:37:42.336 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102114.log 2024-09-14T07:37:42.338 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.127008.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74887.log.gz 2024-09-14T07:37:42.338 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85379.log.gz 2024-09-14T07:37:42.338 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65398.log 2024-09-14T07:37:42.339 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102114.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127008.log.gz 2024-09-14T07:37:42.339 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102114.log.gz 2024-09-14T07:37:42.339 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82796.log.gz 2024-09-14T07:37:42.340 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83134.log.gz 2024-09-14T07:37:42.340 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124806.log 2024-09-14T07:37:42.341 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.65398.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77560.log 2024-09-14T07:37:42.341 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55417.log 2024-09-14T07:37:42.342 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124806.log: /var/log/ceph/cluster2-client.mirror.77560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123505.log 2024-09-14T07:37:42.343 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55417.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33744.log.gz 2024-09-14T07:37:42.343 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71200.log 2024-09-14T07:37:42.344 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.123505.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124806.log.gz 2024-09-14T07:37:42.344 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123505.log.gz 2024-09-14T07:37:42.344 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124664.log 2024-09-14T07:37:42.345 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102614.log 2024-09-14T07:37:42.345 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124664.log: /var/log/ceph/cluster2-client.mirror.71200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123296.log 2024-09-14T07:37:42.345 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85207.log.gz 2024-09-14T07:37:42.346 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.102614.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124664.log.gz 2024-09-14T07:37:42.346 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86570.log 2024-09-14T07:37:42.347 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123296.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.102614.log.gz 2024-09-14T07:37:42.347 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98988.log 2024-09-14T07:37:42.348 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123296.log.gz 2024-09-14T07:37:42.348 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68831.log 2024-09-14T07:37:42.348 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104583.log 2024-09-14T07:37:42.349 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.98988.log.gz 2024-09-14T07:37:42.349 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116330.log 2024-09-14T07:37:42.350 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.104583.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104583.log.gz 2024-09-14T07:37:42.350 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132974.log 2024-09-14T07:37:42.351 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116330.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116330.log.gz 2024-09-14T07:37:42.352 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79055.log 2024-09-14T07:37:42.352 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112707.log 2024-09-14T07:37:42.352 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53561.log.gz 2024-09-14T07:37:42.353 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132974.log: /var/log/ceph/cluster2-client.mirror.79055.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.132974.log.gz 2024-09-14T07:37:42.356 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108058.log 2024-09-14T07:37:42.357 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.112707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134106.log 2024-09-14T07:37:42.357 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.89595.log.gz 88.0% 2024-09-14T07:37:42.357 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.112707.log.gz 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108058.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84796.log 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.108058.log.gz 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88046.log 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134106.log: /var/log/ceph/cluster2-client.mirror.84796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87793.log 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134106.log.gz/var/log/ceph/cluster2-client.mirror.88046.log: 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99212.log 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106194.log 2024-09-14T07:37:42.360 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.110149.log.gz 2024-09-14T07:37:42.361 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45932.log 2024-09-14T07:37:42.361 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99212.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99212.log.gz/var/log/ceph/cluster1-client.mirror.106194.log: 2024-09-14T07:37:42.361 INFO:teuthology.orchestra.run.smithi169.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31050.log.gz 2024-09-14T07:37:42.362 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.106194.log.gz 2024-09-14T07:37:42.362 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53693.log 2024-09-14T07:37:42.362 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65264.log 2024-09-14T07:37:42.362 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45932.log: /var/log/ceph/cluster1-client.mirror.53693.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66342.log 2024-09-14T07:37:42.363 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124016.log 2024-09-14T07:37:42.363 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65264.log: /var/log/ceph/cluster2-client.mirror.66342.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55789.log 2024-09-14T07:37:42.364 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97255.log 2024-09-14T07:37:42.366 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124016.log: /var/log/ceph/cluster2-client.mirror.55789.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124016.log.gz 2024-09-14T07:37:42.366 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110488.log 2024-09-14T07:37:42.366 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99649.log 2024-09-14T07:37:42.368 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110488.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.110488.log.gz 2024-09-14T07:37:42.368 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97255.log.gz 2024-09-14T07:37:42.368 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74418.log 2024-09-14T07:37:42.369 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99649.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99649.log.gz 2024-09-14T07:37:42.370 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44956.log 2024-09-14T07:37:42.370 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74418.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33054.log 2024-09-14T07:37:42.371 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127223.log 2024-09-14T07:37:42.371 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104422.log 2024-09-14T07:37:42.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127223.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127223.log.gz 2024-09-14T07:37:42.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132817.log 2024-09-14T07:37:42.373 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107442.log 2024-09-14T07:37:42.373 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132817.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104422.log.gz 2024-09-14T07:37:42.373 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132817.log.gz 2024-09-14T07:37:42.374 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48164.log 2024-09-14T07:37:42.374 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128190.log 2024-09-14T07:37:42.374 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48164.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107442.log.gz 2024-09-14T07:37:42.375 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75324.log 2024-09-14T07:37:42.375 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128190.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122363.log 2024-09-14T07:37:42.375 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75324.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128190.log.gz 2024-09-14T07:37:42.375 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49191.log 2024-09-14T07:37:42.376 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114398.log 2024-09-14T07:37:42.376 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49191.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122363.log.gz 2024-09-14T07:37:42.377 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42038.log 2024-09-14T07:37:42.377 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114398.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30794.log 2024-09-14T07:37:42.378 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53425.log 2024-09-14T07:37:42.378 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30794.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33054.log.gz 2024-09-14T07:37:42.379 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.114398.log.gz 2024-09-14T07:37:42.379 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.46897.log.gz 2024-09-14T07:37:42.379 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85293.log 2024-09-14T07:37:42.380 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84617.log 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.53425.log: /var/log/ceph/cluster1-client.mirror.85293.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56440.log.gz 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128618.log 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60538.log.gz 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60596.log.gz 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79661.log.gz 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84617.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50249.log.gz 2024-09-14T07:37:42.381 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62402.log 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81821.log.gz 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128370.log 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr: 81.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59438.log.gz 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58451.log.gz 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128618.log: /var/log/ceph/cluster2-client.mirror.62402.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59614.log.gz 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.128618.log.gz 2024-09-14T07:37:42.382 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61844.log.gz 2024-09-14T07:37:42.384 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90976.log 2024-09-14T07:37:42.385 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100664.log 2024-09-14T07:37:42.385 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.128370.log.gz/var/log/ceph/cluster2-client.mirror.90976.log: 2024-09-14T07:37:42.388 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57091.log 2024-09-14T07:37:42.389 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100664.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100664.log.gz 2024-09-14T07:37:42.389 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.88286.log 2024-09-14T07:37:42.390 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57091.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52990.log.gz 2024-09-14T07:37:42.392 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55010.log.gz 2024-09-14T07:37:42.392 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51644.log 2024-09-14T07:37:42.400 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.2.88286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97297.log 2024-09-14T07:37:42.401 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51644.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.84268.log.gz 2024-09-14T07:37:42.403 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84882.log.gz 2024-09-14T07:37:42.406 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62309.log.gz 2024-09-14T07:37:42.408 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65287.log 2024-09-14T07:37:42.408 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71200.log.gz 2024-09-14T07:37:42.410 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97297.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.97297.log.gz 2024-09-14T07:37:42.416 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85164.log.gz 2024-09-14T07:37:42.419 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.65398.log.gz 2024-09-14T07:37:42.419 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38584.log.gz 2024-09-14T07:37:42.420 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42689.log.gz 2024-09-14T07:37:42.420 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39797.log.gz 2024-09-14T07:37:42.420 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45932.log.gz 2024-09-14T07:37:42.420 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40918.log 2024-09-14T07:37:42.421 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.65287.log: 54.0% -- replaced with /var/log/ceph/cluster1-client.mirror.30794.log.gz 2024-09-14T07:37:42.428 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119074.log 2024-09-14T07:37:42.428 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1%/var/log/ceph/cluster2-client.mirror.40918.log: -- replaced with /var/log/ceph/cluster2-client.mirror.45186.log.gz 2024-09-14T07:37:42.430 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57091.log.gz 2024-09-14T07:37:42.430 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48199.log.gz 2024-09-14T07:37:42.430 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82629.log.gz 2024-09-14T07:37:42.430 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75324.log.gz 2024-09-14T07:37:42.431 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77560.log.gz 2024-09-14T07:37:42.431 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79055.log.gz 2024-09-14T07:37:42.431 INFO:teuthology.orchestra.run.smithi169.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.2.88286.log.gz 2024-09-14T07:37:42.432 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82730.log.gz 2024-09-14T07:37:42.432 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84617.log.gz 2024-09-14T07:37:42.432 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67220.log 2024-09-14T07:37:42.433 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119074.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119074.log.gz 2024-09-14T07:37:42.433 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43561.log 2024-09-14T07:37:42.433 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133707.log 2024-09-14T07:37:42.435 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87293.log 2024-09-14T07:37:42.435 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133707.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133707.log.gz 2024-09-14T07:37:42.436 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94534.log 2024-09-14T07:37:42.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99784.log 2024-09-14T07:37:42.439 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94534.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94534.log.gz 2024-09-14T07:37:42.439 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127583.log 2024-09-14T07:37:42.440 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99784.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99784.log.gz 2024-09-14T07:37:42.440 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108384.log 2024-09-14T07:37:42.441 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127583.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127583.log.gz 2024-09-14T07:37:42.441 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128325.log 2024-09-14T07:37:42.442 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.108384.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.108384.log.gz 2024-09-14T07:37:42.443 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78550.log 2024-09-14T07:37:42.443 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128325.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128325.log.gz 2024-09-14T07:37:42.444 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113081.log 2024-09-14T07:37:42.452 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78550.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132920.log 2024-09-14T07:37:42.453 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113081.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113081.log.gz 2024-09-14T07:37:42.456 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129315.log 2024-09-14T07:37:42.456 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132920.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132920.log.gz 2024-09-14T07:37:42.457 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56963.log 2024-09-14T07:37:42.457 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129315.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129315.log.gz 2024-09-14T07:37:42.457 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93815.log 2024-09-14T07:37:42.458 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52295.log 2024-09-14T07:37:42.459 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93815.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93815.log.gz 2024-09-14T07:37:42.459 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51923.log 2024-09-14T07:37:42.460 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52798.log 2024-09-14T07:37:42.460 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51923.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124348.log 2024-09-14T07:37:42.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52798.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94753.log 2024-09-14T07:37:42.463 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124348.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124348.log.gz 2024-09-14T07:37:42.463 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91288.log 2024-09-14T07:37:42.468 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94753.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.94753.log.gz 2024-09-14T07:37:42.469 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39425.log 2024-09-14T07:37:42.469 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91288.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91288.log.gz 2024-09-14T07:37:42.469 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74511.log 2024-09-14T07:37:42.470 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94728.log 2024-09-14T07:37:42.471 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76479.log 2024-09-14T07:37:42.472 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.94728.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.94728.log.gz 2024-09-14T07:37:42.473 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88753.log 2024-09-14T07:37:42.473 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86019.log 2024-09-14T07:37:42.474 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88753.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88753.log.gz 2024-09-14T07:37:42.474 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102029.log 2024-09-14T07:37:42.474 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77095.log 2024-09-14T07:37:42.475 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102029.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.102029.log.gz 2024-09-14T07:37:42.476 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116533.log 2024-09-14T07:37:42.476 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65219.log 2024-09-14T07:37:42.477 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116533.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.116533.log.gz 2024-09-14T07:37:42.478 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94580.log 2024-09-14T07:37:42.478 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.65219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32438.log 2024-09-14T07:37:42.479 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94580.log: 88.2% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55847.log.gz 2024-09-14T07:37:42.479 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.94580.log.gz 2024-09-14T07:37:42.479 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33275.log.gz 2024-09-14T07:37:42.480 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56068.log.gz 2024-09-14T07:37:42.480 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101311.log 2024-09-14T07:37:42.481 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62774.log 2024-09-14T07:37:42.481 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101311.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101311.log.gz 2024-09-14T07:37:42.482 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87649.log 2024-09-14T07:37:42.483 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62774.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50342.log.gz 2024-09-14T07:37:42.485 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62402.log.gz 2024-09-14T07:37:42.485 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40918.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.51644.log.gz 2024-09-14T07:37:42.485 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.485 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60131.log.gz 2024-09-14T07:37:42.485 INFO:teuthology.orchestra.run.smithi169.stderr: 73.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65287.log.gz 2024-09-14T07:37:42.486 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62774.log.gz 2024-09-14T07:37:42.486 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.30455.log 2024-09-14T07:37:42.489 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87649.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84945.log.gz 2024-09-14T07:37:42.494 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131586.log 2024-09-14T07:37:42.496 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.2.30455.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72932.log.gz 2024-09-14T07:37:42.499 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32066.log.gz 2024-09-14T07:37:42.499 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32438.log.gz 2024-09-14T07:37:42.500 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34209.log.gz 2024-09-14T07:37:42.500 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83437.log.gz 2024-09-14T07:37:42.502 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42410.log.gz 2024-09-14T07:37:42.503 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42503.log.gz 2024-09-14T07:37:42.503 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42445.log.gz 2024-09-14T07:37:42.503 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39425.log.gz 2024-09-14T07:37:42.504 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40193.log.gz 2024-09-14T07:37:42.505 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40270.log.gz 2024-09-14T07:37:42.505 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52295.log.gz 2024-09-14T07:37:42.505 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86547.log.gz 2024-09-14T07:37:42.506 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87132.log.gz 2024-09-14T07:37:42.506 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35698.log.gz 2024-09-14T07:37:42.508 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43561.log.gz 2024-09-14T07:37:42.510 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71014.log.gz 2024-09-14T07:37:42.511 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44863.log.gz 2024-09-14T07:37:42.511 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46618.log 2024-09-14T07:37:42.512 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131586.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131586.log.gz 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67220.log.gz 2024-09-14T07:37:42.513 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.513 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69491.log.gz 2024-09-14T07:37:42.516 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69526.log.gz 2024-09-14T07:37:42.517 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82695.log.gz 2024-09-14T07:37:42.517 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74418.log.gz 2024-09-14T07:37:42.518 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76165.log.gz 2024-09-14T07:37:42.518 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78550.log.gz 2024-09-14T07:37:42.537 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126398.log 2024-09-14T07:37:42.544 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59964.log 2024-09-14T07:37:42.545 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126398.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126398.log.gz 2024-09-14T07:37:42.546 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85121.log.gz 2024-09-14T07:37:42.547 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85144.log.gz 2024-09-14T07:37:42.556 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85422.log.gz 2024-09-14T07:37:42.556 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33837.log.gz 2024-09-14T07:37:42.562 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35477.log.gz 2024-09-14T07:37:42.563 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101843.log 2024-09-14T07:37:42.563 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36628.log.gz 2024-09-14T07:37:42.563 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43712.log.gz 2024-09-14T07:37:42.565 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.59964.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82089.log.gz 2024-09-14T07:37:42.565 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43991.log.gz 2024-09-14T07:37:42.568 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86072.log.gz 2024-09-14T07:37:42.568 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66621.log.gz 2024-09-14T07:37:42.568 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80271.log.gz 2024-09-14T07:37:42.572 INFO:teuthology.orchestra.run.smithi169.stderr: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88046.log.gz 2024-09-14T07:37:42.573 INFO:teuthology.orchestra.run.smithi169.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30405.log.gz 2024-09-14T07:37:42.573 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77653.log 2024-09-14T07:37:42.573 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101843.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.101843.log.gz 2024-09-14T07:37:42.574 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79898.log 2024-09-14T07:37:42.574 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66563.log 2024-09-14T07:37:42.575 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114221.log 2024-09-14T07:37:42.576 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66563.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84710.log 2024-09-14T07:37:42.577 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114221.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114221.log.gz 2024-09-14T07:37:42.577 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94835.log 2024-09-14T07:37:42.578 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57928.log 2024-09-14T07:37:42.578 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94835.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64820.log.gz 2024-09-14T07:37:42.579 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64878.log.gz 2024-09-14T07:37:42.579 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65006.log.gz 2024-09-14T07:37:42.580 INFO:teuthology.orchestra.run.smithi129.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2024-09-14T07:37:42.580 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65064.log.gz 2024-09-14T07:37:42.583 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87959.log.gz 2024-09-14T07:37:42.583 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82291.log 2024-09-14T07:37:42.584 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97198.log 2024-09-14T07:37:42.585 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116448.log 2024-09-14T07:37:42.585 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97198.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97198.log.gz 2024-09-14T07:37:42.586 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47420.log.gz 2024-09-14T07:37:42.586 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56498.log 2024-09-14T07:37:42.587 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116448.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116448.log.gz 2024-09-14T07:37:42.587 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91143.log 2024-09-14T07:37:42.588 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59992.log 2024-09-14T07:37:42.588 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91143.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91143.log.gz 2024-09-14T07:37:42.588 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64971.log 2024-09-14T07:37:42.589 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108016.log 2024-09-14T07:37:42.589 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62681.log.gz 2024-09-14T07:37:42.592 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63704.log 2024-09-14T07:37:42.593 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108016.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108016.log.gz 2024-09-14T07:37:42.593 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124786.log 2024-09-14T07:37:42.594 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63704.log: 95.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75359.log 2024-09-14T07:37:42.595 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.130444.log.gz 2024-09-14T07:37:42.595 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81180.log 2024-09-14T07:37:42.595 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124786.log.gz 2024-09-14T07:37:42.595 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83033.log 2024-09-14T07:37:42.596 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133448.log 2024-09-14T07:37:42.596 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83033.log: /var/log/ceph/cluster2-client.mirror.81180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89349.log 2024-09-14T07:37:42.597 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133448.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133448.log.gz 2024-09-14T07:37:42.597 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40116.log 2024-09-14T07:37:42.598 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60166.log 2024-09-14T07:37:42.598 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89349.log: 84.7%/var/log/ceph/cluster2-client.mirror.40116.log: -- replaced with /var/log/ceph/cluster2-client.mirror.89349.log.gz 2024-09-14T07:37:42.600 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55138.log 2024-09-14T07:37:42.600 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60166.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84859.log 2024-09-14T07:37:42.601 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55138.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31033.log 2024-09-14T07:37:42.601 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84859.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40116.log.gz 2024-09-14T07:37:42.602 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45674.log 2024-09-14T07:37:42.602 INFO:teuthology.orchestra.run.smithi169.stderr: 69.3% -- replaced with /var/log/ceph/cluster1-client.admin.30777.log.gz 2024-09-14T07:37:42.602 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102392.log 2024-09-14T07:37:42.603 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.45674.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77979.log 2024-09-14T07:37:42.604 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39332.log 2024-09-14T07:37:42.604 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.77979.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102392.log.gz 2024-09-14T07:37:42.605 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81619.log 2024-09-14T07:37:42.605 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70270.log.gz 2024-09-14T07:37:42.605 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39332.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64971.log.gz 2024-09-14T07:37:42.605 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62832.log 2024-09-14T07:37:42.605 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43061.log.gz 2024-09-14T07:37:42.605 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59230.log 2024-09-14T07:37:42.606 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100070.log 2024-09-14T07:37:42.606 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62832.log: /var/log/ceph/cluster2-client.mirror.59230.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119820.log 2024-09-14T07:37:42.606 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72865.log 2024-09-14T07:37:42.607 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119820.log: /var/log/ceph/cluster2-client.mirror.100070.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100070.log.gz 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119820.log.gz 2024-09-14T07:37:42.607 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.608 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30896.log 2024-09-14T07:37:42.608 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.65287.log 2024-09-14T07:37:42.609 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30896.log: /var/log/ceph/cluster1-client.mirror.72865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75047.log 2024-09-14T07:37:42.609 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57928.log.gz 2024-09-14T07:37:42.610 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.1.65287.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70793.log 2024-09-14T07:37:42.610 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30828.log 2024-09-14T07:37:42.610 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.75047.log: /var/log/ceph/cluster2-client.mirror.70793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115317.log 2024-09-14T07:37:42.612 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127988.log 2024-09-14T07:37:42.612 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30828.log: /var/log/ceph/cluster2-client.mirror.115317.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.59964.log.gz 2024-09-14T07:37:42.612 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50621.log 2024-09-14T07:37:42.612 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115317.log.gz 2024-09-14T07:37:42.612 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127988.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127988.log.gz 2024-09-14T07:37:42.613 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29852.log 2024-09-14T07:37:42.613 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54859.log 2024-09-14T07:37:42.613 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50621.log: /var/log/ceph/cluster2-client.admin.29852.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29852.log.gz 2024-09-14T07:37:42.614 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118910.log 2024-09-14T07:37:42.615 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48292.log 2024-09-14T07:37:42.615 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54859.log: /var/log/ceph/cluster2-client.mirror.118910.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86097.log.gz 2024-09-14T07:37:42.615 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118910.log.gz 2024-09-14T07:37:42.615 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75882.log.gz 2024-09-14T07:37:42.616 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66563.log.gz 2024-09-14T07:37:42.616 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128303.log 2024-09-14T07:37:42.616 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73616.log 2024-09-14T07:37:42.617 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48292.log: /var/log/ceph/cluster1-client.mirror.128303.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128303.log.gz 2024-09-14T07:37:42.618 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77653.log.gz 2024-09-14T07:37:42.618 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34302.log 2024-09-14T07:37:42.618 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49505.log 2024-09-14T07:37:42.618 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.130051.log 2024-09-14T07:37:42.618 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131983.log 2024-09-14T07:37:42.622 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.130051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123454.log 2024-09-14T07:37:42.622 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.130051.log.gz/var/log/ceph/cluster2-client.mirror.131983.log: 2024-09-14T07:37:42.622 INFO:teuthology.orchestra.run.smithi169.stderr: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.131983.log.gz 2024-09-14T07:37:42.623 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98834.log 2024-09-14T07:37:42.623 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125394.log 2024-09-14T07:37:42.623 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123454.log: /var/log/ceph/cluster1-client.mirror.98834.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123454.log.gz 2024-09-14T07:37:42.624 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.98834.log.gz 2024-09-14T07:37:42.624 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67406.log 2024-09-14T07:37:42.624 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124590.log 2024-09-14T07:37:42.627 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67406.log: /var/log/ceph/cluster2-client.mirror.125394.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125394.log.gz 2024-09-14T07:37:42.627 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83033.log.gz 2024-09-14T07:37:42.627 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112486.log 2024-09-14T07:37:42.627 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62181.log.gz 2024-09-14T07:37:42.627 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74174.log 2024-09-14T07:37:42.628 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124590.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62553.log.gz 2024-09-14T07:37:42.628 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49284.log 2024-09-14T07:37:42.628 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112486.log: /var/log/ceph/cluster2-client.mirror.74174.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112486.log.gz 2024-09-14T07:37:42.629 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124590.log.gz 2024-09-14T07:37:42.629 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127875.log 2024-09-14T07:37:42.630 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118050.log 2024-09-14T07:37:42.630 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49284.log: /var/log/ceph/cluster1-client.mirror.127875.log: gzip -5 86.8% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.127875.log.gz 2024-09-14T07:37:42.631 INFO:teuthology.orchestra.run.smithi169.stderr: /var/log/ceph/cluster2-client.mirror.94094.log 2024-09-14T07:37:42.631 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71293.log.gz 2024-09-14T07:37:42.631 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33647.log.gz 2024-09-14T07:37:42.636 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118050.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72223.log.gz 2024-09-14T07:37:42.636 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54859.log.gz 2024-09-14T07:37:42.637 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32403.log 2024-09-14T07:37:42.640 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94094.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.94094.log.gz 2024-09-14T07:37:42.640 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111971.log 2024-09-14T07:37:42.640 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32403.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89047.log.gz 2024-09-14T07:37:42.641 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111971.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.111971.log.gz 2024-09-14T07:37:42.643 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124235.log 2024-09-14T07:37:42.643 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40825.log.gz 2024-09-14T07:37:42.644 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85746.log 2024-09-14T07:37:42.644 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125518.log 2024-09-14T07:37:42.645 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85746.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124235.log.gz 2024-09-14T07:37:42.645 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118008.log 2024-09-14T07:37:42.645 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49691.log 2024-09-14T07:37:42.646 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118008.log: 89.2% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90976.log.gz 2024-09-14T07:37:42.646 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.125518.log.gz 2024-09-14T07:37:42.646 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72839.log 2024-09-14T07:37:42.647 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49691.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97966.log 2024-09-14T07:37:42.647 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118008.log.gz 2024-09-14T07:37:42.647 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25381.log 2024-09-14T07:37:42.649 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72839.log: 92.9%/var/log/ceph/cluster2-client.mirror.97966.log: -- replaced with /var/log/ceph/cluster2-client.mirror.85015.log.gz 2024-09-14T07:37:42.649 INFO:teuthology.orchestra.run.smithi169.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86019.log.gz 2024-09-14T07:37:42.649 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129383.log 2024-09-14T07:37:42.649 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85293.log.gz 2024-09-14T07:37:42.649 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97966.log.gz 2024-09-14T07:37:42.649 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.25381.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25381.log.gz 2024-09-14T07:37:42.651 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32403.log.gz 2024-09-14T07:37:42.651 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113215.log 2024-09-14T07:37:42.651 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125090.log 2024-09-14T07:37:42.651 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129383.log: /var/log/ceph/cluster2-client.mirror.113215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59803.log 2024-09-14T07:37:42.651 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125090.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129383.log.gz 2024-09-14T07:37:42.651 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66284.log 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34302.log.gz 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.59803.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41569.log 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.39107.log.gz 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66284.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125090.log.gz 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81685.log 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39332.log.gz 2024-09-14T07:37:42.653 INFO:teuthology.orchestra.run.smithi169.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31033.log.gz 2024-09-14T07:37:42.654 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41569.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30828.log.gz 2024-09-14T07:37:42.655 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44956.log.gz 2024-09-14T07:37:42.655 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121236.log 2024-09-14T07:37:42.655 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70642.log 2024-09-14T07:37:42.656 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81685.log: /var/log/ceph/cluster2-client.mirror.121236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51702.log 2024-09-14T07:37:42.656 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121236.log.gz 2024-09-14T07:37:42.656 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89156.log 2024-09-14T07:37:42.657 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93900.log 2024-09-14T07:37:42.657 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51702.log: /var/log/ceph/cluster2-client.mirror.89156.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98281.log 2024-09-14T07:37:42.658 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93900.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93900.log.gz 2024-09-14T07:37:42.658 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42317.log 2024-09-14T07:37:42.659 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98281.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98281.log.gz 2024-09-14T07:37:42.659 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48292.log.gz 2024-09-14T07:37:42.661 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70642.log.gz 2024-09-14T07:37:42.662 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30580.log 2024-09-14T07:37:42.662 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42317.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65199.log 2024-09-14T07:37:42.663 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70828.log.gz 2024-09-14T07:37:42.664 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30580.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87293.log.gz 2024-09-14T07:37:42.664 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126469.log 2024-09-14T07:37:42.665 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54173.log 2024-09-14T07:37:42.666 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126469.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126469.log.gz 2024-09-14T07:37:42.666 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91531.log 2024-09-14T07:37:42.667 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64355.log 2024-09-14T07:37:42.667 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91531.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91531.log.gz 2024-09-14T07:37:42.668 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131564.log 2024-09-14T07:37:42.669 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64355.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44212.log 2024-09-14T07:37:42.670 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131564.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66284.log.gz 2024-09-14T07:37:42.670 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66342.log.gz 2024-09-14T07:37:42.670 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.131564.log.gz 2024-09-14T07:37:42.671 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44305.log 2024-09-14T07:37:42.672 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114373.log 2024-09-14T07:37:42.676 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44305.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74174.log.gz 2024-09-14T07:37:42.676 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92751.log.gz 2024-09-14T07:37:42.676 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73616.log.gz 2024-09-14T07:37:42.676 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108136.log 2024-09-14T07:37:42.677 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.114373.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72095.log 2024-09-14T07:37:42.677 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9%/var/log/ceph/cluster2-client.mirror.108136.log: -- replaced with /var/log/ceph/cluster1-client.mirror.114373.log.gz 2024-09-14T07:37:42.678 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108136.log.gz 2024-09-14T07:37:42.684 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50400.log 2024-09-14T07:37:42.685 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131877.log 2024-09-14T07:37:42.685 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72095.log: /var/log/ceph/cluster2-client.mirror.50400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81483.log 2024-09-14T07:37:42.686 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122949.log 2024-09-14T07:37:42.686 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131877.log: /var/log/ceph/cluster2-client.mirror.81483.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131877.log.gz 2024-09-14T07:37:42.686 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 has been restored 2024-09-14T07:37:42.691 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94835.log.gz 2024-09-14T07:37:42.692 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65819.log 2024-09-14T07:37:42.692 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.122949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101748.log 2024-09-14T07:37:42.693 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123802.log 2024-09-14T07:37:42.693 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113372.log 2024-09-14T07:37:42.695 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101748.log: /var/log/ceph/cluster2-client.mirror.123802.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122949.log.gz 2024-09-14T07:37:42.695 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123802.log.gz 2024-09-14T07:37:42.695 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94240.log 2024-09-14T07:37:42.696 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113372.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113372.log.gz 2024-09-14T07:37:42.696 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.68620.log.gz 2024-09-14T07:37:42.696 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84191.log 2024-09-14T07:37:42.698 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94240.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94240.log.gz 2024-09-14T07:37:42.698 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30007.log 2024-09-14T07:37:42.698 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115712.log 2024-09-14T07:37:42.699 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84191.log: /var/log/ceph/cluster2-client.admin.30007.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30007.log.gz 2024-09-14T07:37:42.699 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90820.log 2024-09-14T07:37:42.699 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96705.log 2024-09-14T07:37:42.700 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115712.log: /var/log/ceph/cluster2-client.mirror.90820.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115712.log.gz 89.1% 2024-09-14T07:37:42.700 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.82291.log.gz 2024-09-14T07:37:42.700 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82493.log.gz 2024-09-14T07:37:42.700 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124186.log 2024-09-14T07:37:42.701 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.96705.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.96705.log.gz 2024-09-14T07:37:42.701 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62460.log 2024-09-14T07:37:42.702 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124186.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124186.log.gz 2024-09-14T07:37:42.702 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113795.log 2024-09-14T07:37:42.703 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62460.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97501.log 2024-09-14T07:37:42.704 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113795.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113795.log.gz 2024-09-14T07:37:42.704 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48757.log 2024-09-14T07:37:42.705 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.97501.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.97501.log.gz 2024-09-14T07:37:42.705 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35756.log 2024-09-14T07:37:42.706 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48757.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110193.log 2024-09-14T07:37:42.707 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40382.log 2024-09-14T07:37:42.708 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110193.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110193.log.gz 2024-09-14T07:37:42.708 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35570.log 2024-09-14T07:37:42.709 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40382.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85746.log.gz 2024-09-14T07:37:42.710 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108093.log 2024-09-14T07:37:42.710 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95492.log 2024-09-14T07:37:42.711 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108093.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.108093.log.gz 2024-09-14T07:37:42.712 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112732.log 2024-09-14T07:37:42.714 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95492.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95492.log.gz 2024-09-14T07:37:42.714 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44084.log 2024-09-14T07:37:42.716 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56498.log.gz 2024-09-14T07:37:42.716 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55754.log.gz 2024-09-14T07:37:42.716 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68241.log.gz 2024-09-14T07:37:42.717 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68831.log.gz 2024-09-14T07:37:42.720 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112732.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77095.log.gz 2024-09-14T07:37:42.721 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112732.log.gz 2024-09-14T07:37:42.722 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.118050.log.gz 2024-09-14T07:37:42.722 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49970.log.gz 2024-09-14T07:37:42.722 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49284.log.gz 2024-09-14T07:37:42.723 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49691.log.gz 2024-09-14T07:37:42.723 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107167.log 2024-09-14T07:37:42.724 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130887.log 2024-09-14T07:37:42.724 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44084.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81685.log.gz 2024-09-14T07:37:42.724 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.107167.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81619.log.gz 2024-09-14T07:37:42.725 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58916.log.gz 2024-09-14T07:37:42.725 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.107167.log.gz 2024-09-14T07:37:42.725 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59230.log.gz 2024-09-14T07:37:42.725 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68334.log 2024-09-14T07:37:42.725 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130887.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62460.log.gz 2024-09-14T07:37:42.726 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72095.log.gz 2024-09-14T07:37:42.726 INFO:teuthology.orchestra.run.smithi169.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.130887.log.gz 2024-09-14T07:37:42.727 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107811.log 2024-09-14T07:37:42.727 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68334.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84796.log.gz 2024-09-14T07:37:42.729 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111002.log 2024-09-14T07:37:42.729 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85187.log.gz 2024-09-14T07:37:42.730 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107811.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107811.log.gz 2024-09-14T07:37:42.730 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38177.log 2024-09-14T07:37:42.730 INFO:teuthology.orchestra.run.smithi169.stderr: 74.2% -- replaced with /var/log/ceph/cluster1-client.mirror.30580.log.gz 2024-09-14T07:37:42.737 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111002.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51702.log.gz 2024-09-14T07:37:42.737 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.111002.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81180.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121003.log 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133386.log 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38177.log: /var/log/ceph/cluster2-client.mirror.121003.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41569.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42596.log 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63204.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133386.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42317.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42038.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44084.log.gz 2024-09-14T07:37:42.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133386.log.gz 2024-09-14T07:37:42.739 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63146.log 2024-09-14T07:37:42.739 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44305.log.gz 2024-09-14T07:37:42.739 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112963.log 2024-09-14T07:37:42.739 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72130.log 2024-09-14T07:37:42.740 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112963.log: 81.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31834.log.gz 2024-09-14T07:37:42.740 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.45674.log.gz 2024-09-14T07:37:42.741 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57277.log.gz 2024-09-14T07:37:42.751 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63704.log.gz 2024-09-14T07:37:42.753 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73058.log 2024-09-14T07:37:42.753 INFO:teuthology.orchestra.run.smithi169.stderr: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87649.log.gz 2024-09-14T07:37:42.753 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116141.log 2024-09-14T07:37:42.753 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73058.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72839.log.gz 2024-09-14T07:37:42.754 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134058.log 2024-09-14T07:37:42.754 INFO:teuthology.orchestra.run.smithi169.stderr: 81.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72865.log.gz 2024-09-14T07:37:42.754 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116141.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64355.log.gz 2024-09-14T07:37:42.755 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78686.log.gz 2024-09-14T07:37:42.756 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.116141.log.gz 2024-09-14T07:37:42.756 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84191.log.gz 2024-09-14T07:37:42.757 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84773.log 2024-09-14T07:37:42.757 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134058.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134058.log.gz 2024-09-14T07:37:42.758 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103280.log 2024-09-14T07:37:42.758 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84773.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48757.log.gz 2024-09-14T07:37:42.759 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84652.log 2024-09-14T07:37:42.760 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103280.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.103280.log.gz 2024-09-14T07:37:42.760 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102592.log 2024-09-14T07:37:42.761 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84652.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121636.log 2024-09-14T07:37:42.763 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102592.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102592.log.gz 2024-09-14T07:37:42.763 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114847.log 2024-09-14T07:37:42.766 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121636.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121636.log.gz 2024-09-14T07:37:42.766 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43433.log 2024-09-14T07:37:42.775 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114847.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87793.log.gz 2024-09-14T07:37:42.776 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124693.log 2024-09-14T07:37:42.776 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43433.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50028.log 2024-09-14T07:37:42.777 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124693.log: 91.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.30480.log.gz 2024-09-14T07:37:42.777 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124693.log.gz 2024-09-14T07:37:42.778 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133365.log 2024-09-14T07:37:42.778 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98586.log 2024-09-14T07:37:42.779 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133365.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133365.log.gz 2024-09-14T07:37:42.780 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90567.log 2024-09-14T07:37:42.781 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98586.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98586.log.gz 2024-09-14T07:37:42.781 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126002.log 2024-09-14T07:37:42.792 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42875.log 2024-09-14T07:37:42.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126002.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126002.log.gz 2024-09-14T07:37:42.796 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.32996.log.gz 2024-09-14T07:37:42.801 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46618.log.gz 2024-09-14T07:37:42.801 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54173.log.gz 2024-09-14T07:37:42.801 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55789.log.gz 2024-09-14T07:37:42.801 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68334.log.gz 2024-09-14T07:37:42.801 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76479.log.gz 2024-09-14T07:37:42.803 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75135.log.gz 2024-09-14T07:37:42.804 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72130.log.gz 88.3% 2024-09-14T07:37:42.804 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.50621.log.gz 2024-09-14T07:37:42.804 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55417.log.gz 2024-09-14T07:37:42.805 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.49505.log.gz 2024-09-14T07:37:42.805 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60166.log.gz 2024-09-14T07:37:42.807 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79898.log.gz 2024-09-14T07:37:42.807 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126895.log 2024-09-14T07:37:42.808 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41852.log 2024-09-14T07:37:42.809 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126895.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126895.log.gz 2024-09-14T07:37:42.809 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133728.log 2024-09-14T07:37:42.811 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41852.log: 92.7%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.84859.log.gz -5 2024-09-14T07:37:42.811 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.46211.log 2024-09-14T07:37:42.812 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133728.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38177.log.gz 2024-09-14T07:37:42.812 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.133728.log.gz 93.4% 82.0% -- replaced with /var/log/ceph/cluster2-client.mirror.85316.log.gz 2024-09-14T07:37:42.812 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40613.log.gz 2024-09-14T07:37:42.812 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:42.813 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93865.log 2024-09-14T07:37:42.813 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39483.log.gz 2024-09-14T07:37:42.813 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59044.log 2024-09-14T07:37:42.814 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93865.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93865.log.gz 2024-09-14T07:37:42.814 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41852.log.gz 2024-09-14T07:37:42.815 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42596.log.gz 2024-09-14T07:37:42.816 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42724.log.gz 2024-09-14T07:37:42.816 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42875.log.gz 2024-09-14T07:37:42.823 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113953.log 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59044.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48164.log.gz 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56963.log.gz 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70793.log.gz 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90567.log.gz 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67406.log.gz 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67592.log.gz 2024-09-14T07:37:42.826 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75359.log.gz 2024-09-14T07:37:42.828 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121579.log 2024-09-14T07:37:42.830 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.113953.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59803.log.gz 2024-09-14T07:37:42.830 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113953.log.gz 2024-09-14T07:37:42.830 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126795.log 2024-09-14T07:37:42.831 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121579.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.121579.log.gz 2024-09-14T07:37:42.831 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114696.log 2024-09-14T07:37:42.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126795.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126795.log.gz 2024-09-14T07:37:42.836 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74511.log.gz 2024-09-14T07:37:42.837 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55196.log 2024-09-14T07:37:42.838 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114696.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114696.log.gz 2024-09-14T07:37:42.838 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75047.log.gz 2024-09-14T07:37:42.838 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131462.log 2024-09-14T07:37:42.838 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55196.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83200.log 2024-09-14T07:37:42.839 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131462.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131462.log.gz 2024-09-14T07:37:42.839 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84652.log.gz 2024-09-14T07:37:42.840 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87743.log 2024-09-14T07:37:42.840 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36814.log 2024-09-14T07:37:42.843 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87743.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.65287.log.gz 2024-09-14T07:37:42.846 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43433.log.gz 2024-09-14T07:37:42.846 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.101748.log.gz 2024-09-14T07:37:42.849 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113908.log 2024-09-14T07:37:42.860 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96750.log 2024-09-14T07:37:42.861 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.113908.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113908.log.gz 2024-09-14T07:37:42.868 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99945.log 2024-09-14T07:37:42.869 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.96750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41945.log 2024-09-14T07:37:42.874 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.99945.log: 92.1% 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.99945.log.gz 2024-09-14T07:37:42.874 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.96750.log.gz 2024-09-14T07:37:42.875 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116703.log 2024-09-14T07:37:42.875 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74790.log 2024-09-14T07:37:42.878 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116703.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.116703.log.gz 2024-09-14T07:37:42.879 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90679.log 2024-09-14T07:37:42.879 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116295.log 2024-09-14T07:37:42.882 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90679.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50400.log.gz 2024-09-14T07:37:42.882 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52776.log.gz 2024-09-14T07:37:42.888 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71351.log.gz 2024-09-14T07:37:42.888 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.113215.log.gz 2024-09-14T07:37:42.889 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47513.log 2024-09-14T07:37:42.890 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116295.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.116295.log.gz 2024-09-14T07:37:42.890 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73058.log.gz 2024-09-14T07:37:42.890 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51237.log 2024-09-14T07:37:42.891 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113975.log 2024-09-14T07:37:42.891 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55138.log.gz 2024-09-14T07:37:42.891 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51237.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55196.log.gz 2024-09-14T07:37:42.893 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98424.log 2024-09-14T07:37:42.895 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113975.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32217.log.gz 2024-09-14T07:37:42.897 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121003.log.gz 2024-09-14T07:37:42.900 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35570.log.gz 2024-09-14T07:37:42.900 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38677.log.gz 2024-09-14T07:37:42.900 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40382.log.gz 2024-09-14T07:37:42.900 INFO:teuthology.orchestra.run.smithi169.stderr: 55.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30896.log.gz 2024-09-14T07:37:42.900 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120104.log 2024-09-14T07:37:42.902 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.120104.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.120104.log.gz 2024-09-14T07:37:42.903 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98802.log 2024-09-14T07:37:42.903 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.98424.log.gz 2024-09-14T07:37:42.903 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.112963.log.gz 2024-09-14T07:37:42.903 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134171.log 2024-09-14T07:37:42.905 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38456.log 2024-09-14T07:37:42.905 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98802.log: /var/log/ceph/cluster2-client.admin.134171.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134171.log.gz 2024-09-14T07:37:42.905 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98802.log.gz 2024-09-14T07:37:42.905 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84071.log 2024-09-14T07:37:42.905 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117673.log 2024-09-14T07:37:42.906 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84071.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98172.log 2024-09-14T07:37:42.906 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117673.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117673.log.gz 2024-09-14T07:37:42.911 INFO:teuthology.orchestra.run.smithi169.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53425.log.gz 2024-09-14T07:37:42.912 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127560.log 2024-09-14T07:37:42.912 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.98172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130936.log 2024-09-14T07:37:42.913 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127560.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127560.log.gz 2024-09-14T07:37:42.913 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73151.log 2024-09-14T07:37:42.914 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130936.log: 83.6% -- replaced with /var/log/ceph/cluster2-client.mirror.130936.log.gz 2024-09-14T07:37:42.914 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.98172.log.gz 2024-09-14T07:37:42.914 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61937.log 2024-09-14T07:37:42.915 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97230.log 2024-09-14T07:37:42.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73151.log: /var/log/ceph/cluster2-client.mirror.61937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73488.log 2024-09-14T07:37:42.916 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.97230.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.97230.log.gz 2024-09-14T07:37:42.918 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102773.log 2024-09-14T07:37:42.918 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74732.log 2024-09-14T07:37:42.919 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77409.log 2024-09-14T07:37:42.919 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102773.log.gz 2024-09-14T07:37:42.920 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132796.log 2024-09-14T07:37:42.921 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77409.log: /var/log/ceph/cluster1-client.mirror.132796.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132796.log.gz 2024-09-14T07:37:42.921 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61472.log 2024-09-14T07:37:42.921 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133056.log 2024-09-14T07:37:42.921 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70886.log 2024-09-14T07:37:42.923 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65241.log 2024-09-14T07:37:42.923 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133056.log: /var/log/ceph/cluster2-client.mirror.70886.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133056.log.gz 2024-09-14T07:37:42.923 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63518.log 2024-09-14T07:37:42.923 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105876.log 2024-09-14T07:37:42.923 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59691.log 2024-09-14T07:37:42.924 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63518.log: /var/log/ceph/cluster2-client.mirror.105876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85977.log 88.2% 2024-09-14T07:37:42.924 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.105876.log.gz 2024-09-14T07:37:42.925 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56347.log 2024-09-14T07:37:42.925 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59691.log: /var/log/ceph/cluster2-client.mirror.85977.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121783.log 2024-09-14T07:37:42.925 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125660.log 2024-09-14T07:37:42.926 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56347.log: /var/log/ceph/cluster2-client.mirror.121783.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.121783.log.gz 2024-09-14T07:37:42.926 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111936.log 2024-09-14T07:37:42.927 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123681.log 2024-09-14T07:37:42.928 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125660.log: /var/log/ceph/cluster2-client.mirror.111936.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111936.log.gz 2024-09-14T07:37:42.928 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125660.log.gz 2024-09-14T07:37:42.928 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58730.log 2024-09-14T07:37:42.929 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52821.log 2024-09-14T07:37:42.929 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123681.log.gz 2024-09-14T07:37:42.929 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57893.log 2024-09-14T07:37:42.929 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74639.log 2024-09-14T07:37:42.930 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52821.log: /var/log/ceph/cluster2-client.mirror.57893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131822.log 2024-09-14T07:37:42.930 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30939.log 2024-09-14T07:37:42.931 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124470.log 2024-09-14T07:37:42.931 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131822.log: /var/log/ceph/cluster2-client.admin.30939.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30939.log.gz 2024-09-14T07:37:42.931 INFO:teuthology.orchestra.run.smithi169.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.131822.log.gz 2024-09-14T07:37:42.933 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41755.log 2024-09-14T07:37:42.933 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71630.log 2024-09-14T07:37:42.934 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41755.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124470.log.gz 2024-09-14T07:37:42.935 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129270.log 2024-09-14T07:37:42.935 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83200.log.gz 2024-09-14T07:37:42.935 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71630.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43375.log 2024-09-14T07:37:42.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129270.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129270.log.gz 2024-09-14T07:37:42.936 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84071.log.gz 2024-09-14T07:37:42.936 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95801.log 2024-09-14T07:37:42.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43375.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47699.log 2024-09-14T07:37:42.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95801.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95801.log.gz 2024-09-14T07:37:42.938 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42631.log 2024-09-14T07:37:42.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52539.log 2024-09-14T07:37:42.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67464.log 2024-09-14T07:37:42.939 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37523.log 2024-09-14T07:37:42.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89290.log 2024-09-14T07:37:42.941 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47176.log 2024-09-14T07:37:42.942 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89290.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81483.log.gz 2024-09-14T07:37:42.945 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72781.log.gz 2024-09-14T07:37:42.945 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46118.log 2024-09-14T07:37:42.949 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79459.log 2024-09-14T07:37:42.949 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46118.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.30530.log.gz 2024-09-14T07:37:42.949 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112598.log 2024-09-14T07:37:42.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82594.log 2024-09-14T07:37:42.951 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112598.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112598.log.gz 2024-09-14T07:37:42.951 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42224.log 2024-09-14T07:37:42.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113760.log 2024-09-14T07:37:42.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61716.log 2024-09-14T07:37:42.956 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113760.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113760.log.gz 2024-09-14T07:37:42.957 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112864.log 2024-09-14T07:37:42.957 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61716.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30205.log 2024-09-14T07:37:42.961 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112864.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112864.log.gz 2024-09-14T07:37:42.962 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35663.log 2024-09-14T07:37:42.962 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.30205.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30205.log.gz 2024-09-14T07:37:42.963 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34674.log 2024-09-14T07:37:42.963 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35663.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124948.log 2024-09-14T07:37:42.966 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34674.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100341.log 2024-09-14T07:37:42.974 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124948.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124948.log.gz 2024-09-14T07:37:42.975 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56777.log 2024-09-14T07:37:42.984 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.100341.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.100341.log.gz 2024-09-14T07:37:42.986 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49191.log.gz 2024-09-14T07:37:42.986 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49319.log.gz 2024-09-14T07:37:42.987 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50028.log.gz 2024-09-14T07:37:42.988 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33182.log 2024-09-14T07:37:42.988 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56777.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101726.log 2024-09-14T07:37:42.990 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105072.log 2024-09-14T07:37:42.990 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51237.log.gz 2024-09-14T07:37:42.990 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.101726.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61937.log.gz 2024-09-14T07:37:42.991 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71630.log.gz 2024-09-14T07:37:42.991 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47176.log.gz 2024-09-14T07:37:42.992 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47513.log.gz 2024-09-14T07:37:42.992 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47699.log.gz 2024-09-14T07:37:42.992 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62832.log.gz 2024-09-14T07:37:42.993 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.101726.log.gz 2024-09-14T07:37:42.994 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54638.log 2024-09-14T07:37:42.994 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105072.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63146.log.gz 2024-09-14T07:37:42.994 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63518.log.gz 2024-09-14T07:37:42.998 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84710.log.gz 2024-09-14T07:37:42.998 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84773.log.gz 2024-09-14T07:37:42.998 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79459.log.gz 2024-09-14T07:37:42.998 INFO:teuthology.orchestra.run.smithi169.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84291.log.gz 2024-09-14T07:37:42.998 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37523.log.gz 2024-09-14T07:37:42.999 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38456.log.gz 2024-09-14T07:37:43.000 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46211.log.gz 2024-09-14T07:37:43.000 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52388.log.gz 2024-09-14T07:37:43.000 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46118.log.gz 2024-09-14T07:37:43.001 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42224.log.gz 2024-09-14T07:37:43.002 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35756.log.gz 2024-09-14T07:37:43.002 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41755.log.gz 2024-09-14T07:37:43.002 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36814.log.gz 2024-09-14T07:37:43.002 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41945.log.gz 2024-09-14T07:37:43.006 INFO:teuthology.orchestra.run.smithi169.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.30580.log.gz 2024-09-14T07:37:43.007 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56777.log.gz 2024-09-14T07:37:43.008 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82594.log.gz 2024-09-14T07:37:43.008 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42631.log.gz 2024-09-14T07:37:43.009 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69584.log.gz 2024-09-14T07:37:43.009 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77409.log.gz 2024-09-14T07:37:43.010 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.31834.log.gz 2024-09-14T07:37:43.010 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73488.log.gz 2024-09-14T07:37:43.012 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74639.log.gz 2024-09-14T07:37:43.013 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74732.log.gz 2024-09-14T07:37:43.013 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74790.log.gz 2024-09-14T07:37:43.017 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.114847.log.gz 2024-09-14T07:37:43.017 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.105072.log.gz 2024-09-14T07:37:43.017 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59044.log.gz 2024-09-14T07:37:43.018 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61096.log 2024-09-14T07:37:43.021 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54638.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85977.log.gz 2024-09-14T07:37:43.032 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63425.log 2024-09-14T07:37:43.033 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61096.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.113975.log.gz 2024-09-14T07:37:43.034 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127335.log 2024-09-14T07:37:43.035 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63425.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79762.log 2024-09-14T07:37:43.036 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127335.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127335.log.gz 2024-09-14T07:37:43.036 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42817.log 2024-09-14T07:37:43.037 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79762.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90679.log.gzgzip 2024-09-14T07:37:43.037 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60259.log 2024-09-14T07:37:43.038 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108981.log 2024-09-14T07:37:43.039 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42817.log: /var/log/ceph/cluster2-client.mirror.60259.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.83875.log.gz 2024-09-14T07:37:43.039 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105500.log 2024-09-14T07:37:43.039 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.108981.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41887.log 2024-09-14T07:37:43.039 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85935.log 2024-09-14T07:37:43.040 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105500.log: /var/log/ceph/cluster2-client.mirror.41887.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105500.log.gz 2024-09-14T07:37:43.040 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33895.log 2024-09-14T07:37:43.041 INFO:teuthology.orchestra.run.smithi169.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.108981.log.gz 2024-09-14T07:37:43.041 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104196.log 2024-09-14T07:37:43.041 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85935.log: /var/log/ceph/cluster2-client.mirror.33895.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72409.log 2024-09-14T07:37:43.041 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104196.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69374.log 2024-09-14T07:37:43.043 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104196.log.gz 2024-09-14T07:37:43.043 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72409.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87743.log.gz 2024-09-14T07:37:43.043 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111795.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35198.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37058.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37933.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111795.log.gz 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37058.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109917.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39297.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32810.log 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39297.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109917.log.gz 2024-09-14T07:37:43.044 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104899.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32810.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30505.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104899.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133899.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30505.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.104899.log.gz 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126244.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133899.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76103.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133899.log.gz 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124328.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76103.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133684.log 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124328.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126244.log.gz 2024-09-14T07:37:43.048 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62216.log 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39946.log 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124328.log.gz 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62216.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133684.log.gz 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111830.log 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.39946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127425.log 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107939.log 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127425.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111830.log.gz 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133769.log 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127425.log.gz 2024-09-14T07:37:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58823.log 2024-09-14T07:37:43.050 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107939.log: /var/log/ceph/cluster2-client.mirror.133769.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107939.log.gz 2024-09-14T07:37:43.050 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.133769.log.gz 2024-09-14T07:37:43.052 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124715.log 2024-09-14T07:37:43.052 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134405.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118680.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124715.log: /var/log/ceph/cluster2-client.admin.134405.log: gzip 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134405.log.gz -5 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.128933.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124715.log.gz 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98767.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128933.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128280.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128933.log.gz 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67871.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126753.log 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128280.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98767.log.gz 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0%/var/log/ceph/cluster2-client.mirror.67871.log: -- replaced with /var/log/ceph/cluster1-client.mirror.128280.log.gz 2024-09-14T07:37:43.055 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122837.log 2024-09-14T07:37:43.056 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72560.log 2024-09-14T07:37:43.056 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126753.log: /var/log/ceph/cluster2-client.mirror.122837.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112255.log 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123154.log 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122837.log.gz 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42073.log 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.123154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119120.log 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112255.log.gz 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.126753.log.gz 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42073.log: gzip 88.0% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.123154.log.gz --verbose 2024-09-14T07:37:43.057 INFO:teuthology.orchestra.run.smithi169.stderr: -- /var/log/ceph/cluster2-client.mirror.97568.log 2024-09-14T07:37:43.060 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94978.log 2024-09-14T07:37:43.060 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119120.log: /var/log/ceph/cluster2-client.mirror.97568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58637.log 2024-09-14T07:37:43.061 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94978.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91867.log 2024-09-14T07:37:43.061 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119120.log.gz 2024-09-14T07:37:43.062 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58637.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94978.log.gz 2024-09-14T07:37:43.062 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44270.log 2024-09-14T07:37:43.062 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123267.log 2024-09-14T07:37:43.062 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.91867.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90341.log 2024-09-14T07:37:43.064 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123267.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.123267.log.gz 2024-09-14T07:37:43.064 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109064.log 2024-09-14T07:37:43.065 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44270.log: /var/log/ceph/cluster2-client.mirror.90341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42131.log 2024-09-14T07:37:43.065 INFO:teuthology.orchestra.run.smithi169.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118680.log.gz 2024-09-14T07:37:43.065 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109064.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.102285.log 2024-09-14T07:37:43.066 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.109064.log.gz/var/log/ceph/cluster2-client.mirror.42131.log: 2024-09-14T07:37:43.066 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71944.log 2024-09-14T07:37:43.066 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.102285.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62367.log 2024-09-14T07:37:43.066 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73151.log.gz 2024-09-14T07:37:43.067 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71944.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91867.log.gz 2024-09-14T07:37:43.068 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88906.log 2024-09-14T07:37:43.068 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62367.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59891.log 2024-09-14T07:37:43.069 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57335.log 2024-09-14T07:37:43.069 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88906.log: /var/log/ceph/cluster2-client.mirror.59891.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90820.log.gz 2024-09-14T07:37:43.070 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69770.log 2024-09-14T07:37:43.070 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46060.log 2024-09-14T07:37:43.070 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69770.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88639.log 2024-09-14T07:37:43.070 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107302.log 2024-09-14T07:37:43.071 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59726.log 2024-09-14T07:37:43.071 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107302.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107302.log.gz 2024-09-14T07:37:43.072 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130755.log 2024-09-14T07:37:43.072 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69352.log 2024-09-14T07:37:43.073 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130755.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130755.log.gz 2024-09-14T07:37:43.074 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66063.log 2024-09-14T07:37:43.074 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98194.log 2024-09-14T07:37:43.075 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66063.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89156.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111600.log 2024-09-14T07:37:43.075 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:43.075 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98194.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108961.log 2024-09-14T07:37:43.076 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111600.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111600.log.gz 2024-09-14T07:37:43.077 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55975.log 2024-09-14T07:37:43.077 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33182.log.gz 2024-09-14T07:37:43.078 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108961.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.30455.log.gz 2024-09-14T07:37:43.078 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56347.log.gz 2024-09-14T07:37:43.079 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.108961.log.gz 2024-09-14T07:37:43.079 INFO:teuthology.orchestra.run.smithi169.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.65219.log.gz 2024-09-14T07:37:43.080 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130356.log 2024-09-14T07:37:43.080 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65819.log.gz 2024-09-14T07:37:43.080 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55975.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58730.log.gz 2024-09-14T07:37:43.080 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41662.log 2024-09-14T07:37:43.081 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130356.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.130356.log.gz 2024-09-14T07:37:43.082 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94499.log 2024-09-14T07:37:43.083 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94661.log 2024-09-14T07:37:43.083 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94499.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.94499.log.gz 2024-09-14T07:37:43.083 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80065.log 2024-09-14T07:37:43.084 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94661.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.94661.log.gz 2024-09-14T07:37:43.084 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48257.log 2024-09-14T07:37:43.085 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80065.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44549.log 2024-09-14T07:37:43.086 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48257.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85488.log 2024-09-14T07:37:43.086 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32624.log 2024-09-14T07:37:43.087 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55661.log 2024-09-14T07:37:43.088 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60724.log 2024-09-14T07:37:43.088 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55661.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82427.log 2024-09-14T07:37:43.092 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60724.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52879.log 2024-09-14T07:37:43.092 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82427.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72560.log.gz 2024-09-14T07:37:43.093 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61472.log.gz 2024-09-14T07:37:43.094 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32810.log.gz 2024-09-14T07:37:43.094 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33895.log.gz 2024-09-14T07:37:43.096 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66249.log 2024-09-14T07:37:43.097 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.52879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120961.log 2024-09-14T07:37:43.097 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108904.log 2024-09-14T07:37:43.099 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120961.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39988.log.gz 2024-09-14T07:37:43.099 INFO:teuthology.orchestra.run.smithi169.stderr: 81.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39946.log.gz 2024-09-14T07:37:43.100 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.120961.log.gz 2024-09-14T07:37:43.100 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.133800.log 2024-09-14T07:37:43.101 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108904.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108904.log.gz 2024-09-14T07:37:43.101 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32031.log 2024-09-14T07:37:43.103 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.133800.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61096.log.gz 2024-09-14T07:37:43.103 INFO:teuthology.orchestra.run.smithi169.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.133800.log.gz 2024-09-14T07:37:43.103 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52539.log.gz 2024-09-14T07:37:43.104 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103548.log 2024-09-14T07:37:43.104 INFO:teuthology.orchestra.run.smithi169.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.102285.log.gz 2024-09-14T07:37:43.104 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107904.log 2024-09-14T07:37:43.104 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103548.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.103548.log.gz 2024-09-14T07:37:43.105 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59691.log.gz 2024-09-14T07:37:43.105 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59726.log.gz 2024-09-14T07:37:43.106 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61751.log.gz 2024-09-14T07:37:43.106 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61716.log.gz 2024-09-14T07:37:43.106 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.52879.log.gz 2024-09-14T07:37:43.107 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35663.log.gz 2024-09-14T07:37:43.108 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41662.log.gz 2024-09-14T07:37:43.109 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.41887.log.gz 2024-09-14T07:37:43.109 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43375.log.gz 2024-09-14T07:37:43.109 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44212.log.gz 2024-09-14T07:37:43.110 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57893.log.gz 2024-09-14T07:37:43.110 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70886.log.gz 2024-09-14T07:37:43.110 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82427.log.gz 2024-09-14T07:37:43.114 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77979.log.gz 2024-09-14T07:37:43.114 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44270.log.gz 2024-09-14T07:37:43.116 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55882.log 2024-09-14T07:37:43.117 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107904.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66249.log.gz 2024-09-14T07:37:43.117 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107904.log.gz 2024-09-14T07:37:43.118 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67871.log.gz 2024-09-14T07:37:43.118 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69770.log.gz 2024-09-14T07:37:43.118 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76103.log.gz 2024-09-14T07:37:43.121 INFO:teuthology.orchestra.run.smithi169.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.83875.log.gz 2024-09-14T07:37:43.121 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76816.log 2024-09-14T07:37:43.122 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55882.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51923.log.gz 2024-09-14T07:37:43.124 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43468.log 2024-09-14T07:37:43.124 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76068.log 2024-09-14T07:37:43.128 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43468.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85935.log.gz 2024-09-14T07:37:43.129 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87019.log 2024-09-14T07:37:43.129 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68738.log 2024-09-14T07:37:43.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134223.log 2024-09-14T07:37:43.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132457.log 2024-09-14T07:37:43.130 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125919.log 2024-09-14T07:37:43.130 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134223.log.gz 2024-09-14T07:37:43.136 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78282.log 2024-09-14T07:37:43.147 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125919.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125919.log.gz 2024-09-14T07:37:43.148 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94372.log 2024-09-14T07:37:43.150 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64913.log 2024-09-14T07:37:43.150 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94372.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94372.log.gz 2024-09-14T07:37:43.153 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115874.log 2024-09-14T07:37:43.153 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58265.log 2024-09-14T07:37:43.154 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115874.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115874.log.gz 2024-09-14T07:37:43.154 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66999.log 2024-09-14T07:37:43.158 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58265.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131847.log 2024-09-14T07:37:43.159 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47978.log 2024-09-14T07:37:43.163 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131847.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.131847.log.gz 2024-09-14T07:37:43.163 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123925.log 2024-09-14T07:37:43.164 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40228.log 2024-09-14T07:37:43.165 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123925.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123925.log.gz 2024-09-14T07:37:43.165 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48815.log 2024-09-14T07:37:43.167 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40228.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78247.log 2024-09-14T07:37:43.168 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37093.log 2024-09-14T07:37:43.168 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120694.log 2024-09-14T07:37:43.169 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37093.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68570.log 2024-09-14T07:37:43.170 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120694.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120694.log.gz 2024-09-14T07:37:43.170 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68055.log 2024-09-14T07:37:43.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.68570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.53425.log 2024-09-14T07:37:43.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68055.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85336.log 2024-09-14T07:37:43.172 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.3.53425.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130619.log 2024-09-14T07:37:43.173 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99742.log 2024-09-14T07:37:43.175 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130619.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130619.log.gz 2024-09-14T07:37:43.175 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124377.log 2024-09-14T07:37:43.178 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99742.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99742.log.gz 2024-09-14T07:37:43.178 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30305.log 2024-09-14T07:37:43.179 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124377.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124377.log.gz 2024-09-14T07:37:43.180 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44828.log 2024-09-14T07:37:43.180 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30305.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30305.log.gz 2024-09-14T07:37:43.181 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97850.log 2024-09-14T07:37:43.181 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60782.log 2024-09-14T07:37:43.182 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97850.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97850.log.gz 2024-09-14T07:37:43.185 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95071.log 2024-09-14T07:37:43.186 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60782.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.68570.log.gz 2024-09-14T07:37:43.186 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118767.log 2024-09-14T07:37:43.187 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95071.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95071.log.gz 2024-09-14T07:37:43.188 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97469.log 2024-09-14T07:37:43.188 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118767.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118767.log.gz 2024-09-14T07:37:43.189 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65131.log 2024-09-14T07:37:43.189 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97469.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97469.log.gz 2024-09-14T07:37:43.190 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98459.log 2024-09-14T07:37:43.190 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.65131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29929.log 2024-09-14T07:37:43.191 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98459.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.98459.log.gz 2024-09-14T07:37:43.191 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52848.log 2024-09-14T07:37:43.192 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29929.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29929.log.gz 2024-09-14T07:37:43.192 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58637.log.gz 2024-09-14T07:37:43.192 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85578.log 2024-09-14T07:37:43.193 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71944.log.gz 2024-09-14T07:37:43.193 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.52848.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45819.log 2024-09-14T07:37:43.194 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85578.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54638.log.gz 2024-09-14T07:37:43.194 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102942.log 2024-09-14T07:37:43.194 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66063.log.gz 2024-09-14T07:37:43.195 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.45819.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62216.log.gz 2024-09-14T07:37:43.195 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63425.log.gz 2024-09-14T07:37:43.195 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85535.log 2024-09-14T07:37:43.197 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102942.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.102942.log.gz 2024-09-14T07:37:43.198 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124448.log 2024-09-14T07:37:43.198 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85535.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32031.log.gz 2024-09-14T07:37:43.199 INFO:teuthology.orchestra.run.smithi169.stderr:gzip 88.0% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.35198.log.gz --verbose 2024-09-14T07:37:43.199 INFO:teuthology.orchestra.run.smithi169.stderr: -- /var/log/ceph/cluster2-client.mirror.32159.log 2024-09-14T07:37:43.200 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124448.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.97568.log.gz 2024-09-14T07:37:43.200 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124448.log.gz 2024-09-14T07:37:43.201 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75157.log 2024-09-14T07:37:43.201 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32159.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98621.log 2024-09-14T07:37:43.201 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37933.log.gz 2024-09-14T07:37:43.202 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103778.log 2024-09-14T07:37:43.202 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98621.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65131.log.gz 2024-09-14T07:37:43.203 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98621.log.gz 2024-09-14T07:37:43.203 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.52848.log.gz 2024-09-14T07:37:43.203 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47827.log 2024-09-14T07:37:43.204 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115955.log 2024-09-14T07:37:43.205 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.103778.log: /var/log/ceph/cluster2-client.mirror.47827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85230.log 2024-09-14T07:37:43.205 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103778.log.gz/var/log/ceph/cluster2-client.mirror.115955.log: 2024-09-14T07:37:43.205 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.115955.log.gz 2024-09-14T07:37:43.205 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48943.log 2024-09-14T07:37:43.206 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50493.log 2024-09-14T07:37:43.206 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85230.log: /var/log/ceph/cluster2-client.mirror.48943.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128775.log 2024-09-14T07:37:43.207 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46676.log 2024-09-14T07:37:43.207 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50493.log: /var/log/ceph/cluster1-client.mirror.128775.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101028.log 2024-09-14T07:37:43.207 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.128775.log.gz 2024-09-14T07:37:43.207 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46676.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125960.log 2024-09-14T07:37:43.208 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.101028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72188.log 2024-09-14T07:37:43.209 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125960.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125960.log.gz 2024-09-14T07:37:43.209 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42073.log.gz 2024-09-14T07:37:43.209 INFO:teuthology.orchestra.run.smithi169.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101028.log.gz 2024-09-14T07:37:43.209 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42131.log.gz 2024-09-14T07:37:43.210 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42817.log.gz 2024-09-14T07:37:43.210 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107496.log 2024-09-14T07:37:43.210 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110709.log 2024-09-14T07:37:43.211 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107496.log: /var/log/ceph/cluster2-client.mirror.72188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.39946.log 2024-09-14T07:37:43.211 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107496.log.gz 2024-09-14T07:37:43.212 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.131029.log 2024-09-14T07:37:43.213 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.110709.log: /var/log/ceph/cluster2-client.mirror.1.39946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66528.log 2024-09-14T07:37:43.213 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110709.log.gz 2024-09-14T07:37:43.213 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.3.131029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121730.log 2024-09-14T07:37:43.213 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130567.log 2024-09-14T07:37:43.214 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121730.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57335.log.gz 2024-09-14T07:37:43.215 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37058.log.gz 2024-09-14T07:37:43.216 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37093.log.gz 2024-09-14T07:37:43.218 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66528.log.gz 2024-09-14T07:37:43.219 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67464.log.gz 2024-09-14T07:37:43.219 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76068.log.gz 2024-09-14T07:37:43.225 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80065.log.gz 2024-09-14T07:37:43.228 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.3.131029.log.gz 2024-09-14T07:37:43.232 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127448.log 2024-09-14T07:37:43.232 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69289.log 2024-09-14T07:37:43.233 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130567.log: /var/log/ceph/cluster1-client.mirror.127448.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127448.log.gz 2024-09-14T07:37:43.233 INFO:teuthology.orchestra.run.smithi169.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.130567.log.gz 2024-09-14T07:37:43.234 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60724.log.gz 2024-09-14T07:37:43.234 INFO:teuthology.orchestra.run.smithi129.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2024-09-14T07:37:43.248 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105047.log 2024-09-14T07:37:43.253 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.69289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34767.log 2024-09-14T07:37:43.254 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.105047.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105047.log.gz 2024-09-14T07:37:43.254 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104161.log 2024-09-14T07:37:43.255 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40860.log 2024-09-14T07:37:43.255 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104161.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104161.log.gz 2024-09-14T07:37:43.258 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69805.log 2024-09-14T07:37:43.259 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30430.log 2024-09-14T07:37:43.261 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125232.log 2024-09-14T07:37:43.261 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.30430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56998.log 2024-09-14T07:37:43.264 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125232.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125232.log.gz 2024-09-14T07:37:43.264 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60631.log 2024-09-14T07:37:43.265 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64134.log 2024-09-14T07:37:43.266 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60631.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75092.log 2024-09-14T07:37:43.267 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64134.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33988.log 2024-09-14T07:37:43.267 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.75092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118189.log 2024-09-14T07:37:43.268 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84223.log 2024-09-14T07:37:43.269 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118189.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118189.log.gz 2024-09-14T07:37:43.271 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131612.log 2024-09-14T07:37:43.271 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84223.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128888.log 2024-09-14T07:37:43.280 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112290.log 2024-09-14T07:37:43.281 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128888.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128888.log.gz 2024-09-14T07:37:43.296 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53125.log 2024-09-14T07:37:43.297 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112290.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112290.log.gz 2024-09-14T07:37:43.307 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98194.log.gz 2024-09-14T07:37:43.310 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79595.log 2024-09-14T07:37:43.310 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53125.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60631.log.gz 2024-09-14T07:37:43.310 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108171.log 2024-09-14T07:37:43.311 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59783.log 2024-09-14T07:37:43.311 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60782.log.gz 2024-09-14T07:37:43.312 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108171.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.108171.log.gz 2024-09-14T07:37:43.313 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.69289.log.gz 2024-09-14T07:37:43.313 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117059.log 2024-09-14T07:37:43.313 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72409.log.gz 2024-09-14T07:37:43.313 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59783.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47827.log.gz 2024-09-14T07:37:43.313 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53125.log.gz 2024-09-14T07:37:43.314 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54022.log 2024-09-14T07:37:43.317 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117059.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.117059.log.gz 2024-09-14T07:37:43.317 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54022.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32159.log.gz 2024-09-14T07:37:43.319 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32624.log.gz 2024-09-14T07:37:43.320 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71072.log 2024-09-14T07:37:43.320 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34674.log.gz 2024-09-14T07:37:43.320 INFO:teuthology.orchestra.run.smithi169.stderr: 74.4% -- replaced with /var/log/ceph/cluster1-client.mirror.45819.log.gz 2024-09-14T07:37:43.320 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120784.log 2024-09-14T07:37:43.320 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46060.log.gz 2024-09-14T07:37:43.324 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71072.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43468.log.gz 2024-09-14T07:37:43.325 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115429.log 2024-09-14T07:37:43.326 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120784.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.120784.log.gz 2024-09-14T07:37:43.326 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40439.log 2024-09-14T07:37:43.327 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115429.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115429.log.gz 2024-09-14T07:37:43.327 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123179.log 2024-09-14T07:37:43.328 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128460.log 2024-09-14T07:37:43.330 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123179.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.123179.log.gz 2024-09-14T07:37:43.331 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59826.log 2024-09-14T07:37:43.331 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128460.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128460.log.gz 2024-09-14T07:37:43.331 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99333.log 2024-09-14T07:37:43.332 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59826.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48257.log.gz 2024-09-14T07:37:43.332 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66999.log.gz 2024-09-14T07:37:43.333 INFO:teuthology.orchestra.run.smithi169.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85578.log.gz 2024-09-14T07:37:43.333 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59137.log 2024-09-14T07:37:43.339 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.99333.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121730.log.gz 2024-09-14T07:37:43.339 INFO:teuthology.orchestra.run.smithi169.stderr: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.99333.log.gz 2024-09-14T07:37:43.339 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107585.log 2024-09-14T07:37:43.339 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74325.log 2024-09-14T07:37:43.340 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107585.log: /var/log/ceph/cluster2-client.mirror.59137.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121271.log 2024-09-14T07:37:43.340 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.107585.log.gz 2024-09-14T07:37:43.340 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116161.log 2024-09-14T07:37:43.341 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74325.log: /var/log/ceph/cluster2-client.mirror.121271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82124.log 2024-09-14T07:37:43.343 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9%/var/log/ceph/cluster1-client.mirror.116161.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.116161.log.gz 2024-09-14T07:37:43.343 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.121271.log.gz 2024-09-14T07:37:43.344 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65877.log 2024-09-14T07:37:43.344 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88616.log 2024-09-14T07:37:43.344 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8%/var/log/ceph/cluster2-client.mirror.82124.log: /var/log/ceph/cluster2-client.mirror.65877.log: -- replaced with /var/log/ceph/cluster2-client.mirror.90341.log.gz 2024-09-14T07:37:43.344 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62739.log 2024-09-14T07:37:43.346 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121143.log 2024-09-14T07:37:43.346 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88616.log: /var/log/ceph/cluster2-client.mirror.62739.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122048.log 2024-09-14T07:37:43.346 INFO:teuthology.orchestra.run.smithi169.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88616.log.gz 2024-09-14T07:37:43.346 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50586.log 2024-09-14T07:37:43.346 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121143.log: /var/log/ceph/cluster2-client.mirror.122048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40732.log 2024-09-14T07:37:43.347 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50586.log: 88.1% 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122048.log.gz 2024-09-14T07:37:43.348 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.121143.log.gz 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32868.log 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112752.log 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72374.log 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32868.log: /var/log/ceph/cluster1-client.mirror.112752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78888.log 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126582.log 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72374.log: /var/log/ceph/cluster2-client.mirror.78888.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85999.log 2024-09-14T07:37:43.349 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126582.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.112752.log.gz 2024-09-14T07:37:43.350 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102512.log 2024-09-14T07:37:43.350 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55289.log 2024-09-14T07:37:43.350 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102512.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102512.log.gz 2024-09-14T07:37:43.353 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109250.log 2024-09-14T07:37:43.353 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126582.log.gz 2024-09-14T07:37:43.353 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117789.log 2024-09-14T07:37:43.354 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64599.log 2024-09-14T07:37:43.354 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109250.log.gz 2024-09-14T07:37:43.354 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87316.log 2024-09-14T07:37:43.354 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117789.log.gz 2024-09-14T07:37:43.354 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126144.log 2024-09-14T07:37:43.355 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62088.log 2024-09-14T07:37:43.355 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93509.log 2024-09-14T07:37:43.355 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93379.log 2024-09-14T07:37:43.356 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62088.log: /var/log/ceph/cluster2-client.mirror.93509.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93509.log.gz 2024-09-14T07:37:43.356 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126144.log.gz 2024-09-14T07:37:43.357 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127113.log 2024-09-14T07:37:43.358 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.93379.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.93379.log.gz 2024-09-14T07:37:43.362 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35977.log 2024-09-14T07:37:43.362 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127113.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108827.log 2024-09-14T07:37:43.362 INFO:teuthology.orchestra.run.smithi169.stderr: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.127113.log.gz 2024-09-14T07:37:43.362 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101909.log 2024-09-14T07:37:43.362 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35977.log: /var/log/ceph/cluster2-client.mirror.108827.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89290.log.gz 2024-09-14T07:37:43.363 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.60014.log 2024-09-14T07:37:43.363 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101909.log: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41794.log 2024-09-14T07:37:43.363 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.108827.log.gz 2024-09-14T07:37:43.363 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.101909.log.gz 2024-09-14T07:37:43.363 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.1.60014.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.89395.log 2024-09-14T07:37:43.363 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39890.log 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.89395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46490.log 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43340.log 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105303.log 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116483.log 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65598.log 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116483.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105303.log.gz 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.116483.log.gz 2024-09-14T07:37:43.365 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79762.log.gz 2024-09-14T07:37:43.368 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116618.log 2024-09-14T07:37:43.368 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128348.log 2024-09-14T07:37:43.369 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128820.log 2024-09-14T07:37:43.370 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128348.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.116618.log.gz 2024-09-14T07:37:43.370 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126195.log 2024-09-14T07:37:43.370 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128348.log.gz 2024-09-14T07:37:43.370 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105775.log 2024-09-14T07:37:43.370 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128820.log: /var/log/ceph/cluster2-client.mirror.126195.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128820.log.gz 2024-09-14T07:37:43.370 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126195.log.gz 2024-09-14T07:37:43.372 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61061.log 2024-09-14T07:37:43.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86239.log 2024-09-14T07:37:43.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61061.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105775.log.gz 2024-09-14T07:37:43.372 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126682.log 2024-09-14T07:37:43.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71479.log 2024-09-14T07:37:43.372 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45016.log 2024-09-14T07:37:43.373 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75638.log 2024-09-14T07:37:43.373 INFO:teuthology.orchestra.run.smithi169.stderr:gzip 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126682.log.gz -5 2024-09-14T07:37:43.373 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.109165.log 2024-09-14T07:37:43.374 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45016.log: /var/log/ceph/cluster2-client.mirror.75638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123996.log 2024-09-14T07:37:43.375 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30999.log 2024-09-14T07:37:43.375 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123996.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.109165.log.gz 2024-09-14T07:37:43.375 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117576.log 2024-09-14T07:37:43.375 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.123996.log.gz 2024-09-14T07:37:43.375 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131921.log 2024-09-14T07:37:43.376 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.117576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69097.log 2024-09-14T07:37:43.376 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131921.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.117576.log.gz 2024-09-14T07:37:43.376 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59288.log 2024-09-14T07:37:43.376 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.131921.log.gz 2024-09-14T07:37:43.382 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79292.log 2024-09-14T07:37:43.382 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2024-09-14T07:37:43.382 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55603.log 2024-09-14T07:37:43.382 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127965.log 2024-09-14T07:37:43.382 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79292.log: /var/log/ceph/cluster2-client.mirror.55603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43526.log 2024-09-14T07:37:43.382 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81922.log 2024-09-14T07:37:43.383 INFO:teuthology.orchestra.run.smithi169.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127965.log.gz 2024-09-14T07:37:43.383 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47548.log 2024-09-14T07:37:43.383 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43526.log: /var/log/ceph/cluster2-client.mirror.81922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49749.log 2024-09-14T07:37:43.384 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85809.log 2024-09-14T07:37:43.384 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47548.log: /var/log/ceph/cluster2-client.mirror.49749.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133001.log 2024-09-14T07:37:43.384 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132002.log 2024-09-14T07:37:43.385 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133001.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.133001.log.gz 2024-09-14T07:37:43.385 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115580.log 2024-09-14T07:37:43.385 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132002.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.132002.log.gz 2024-09-14T07:37:43.386 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108750.log 2024-09-14T07:37:43.386 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115580.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115580.log.gz 2024-09-14T07:37:43.387 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69217.log 2024-09-14T07:37:43.387 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50714.log 2024-09-14T07:37:43.388 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.108750.log.gz 2024-09-14T07:37:43.388 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.69217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113193.log 2024-09-14T07:37:43.388 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43189.log 2024-09-14T07:37:43.388 INFO:teuthology.orchestra.run.smithi169.stderr:gzip/var/log/ceph/cluster1-client.mirror.113193.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58358.log 2024-09-14T07:37:43.389 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73302.log 2024-09-14T07:37:43.389 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58358.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75731.log 2024-09-14T07:37:43.390 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73302.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113193.log.gz 2024-09-14T07:37:43.390 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101808.log 2024-09-14T07:37:43.391 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75731.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130912.log 2024-09-14T07:37:43.391 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101808.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101808.log.gz 2024-09-14T07:37:43.392 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63611.log 2024-09-14T07:37:43.392 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93012.log 2024-09-14T07:37:43.392 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.132457.log.gz 2024-09-14T07:37:43.392 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133955.log 2024-09-14T07:37:43.393 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108317.log 2024-09-14T07:37:43.393 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133955.log: /var/log/ceph/cluster2-client.mirror.93012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131962.log 2024-09-14T07:37:43.393 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.133955.log.gz 2024-09-14T07:37:43.393 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93012.log.gz 2024-09-14T07:37:43.393 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108317.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108317.log.gz 2024-09-14T07:37:43.396 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133847.log 2024-09-14T07:37:43.396 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82897.log 2024-09-14T07:37:43.397 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133847.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133847.log.gz 2024-09-14T07:37:43.397 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100983.log 2024-09-14T07:37:43.397 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66755.log 2024-09-14T07:37:43.397 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131962.log.gz 2024-09-14T07:37:43.398 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.100983.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.100983.log.gz 2024-09-14T07:37:43.399 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94275.log 2024-09-14T07:37:43.399 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55975.log.gz 2024-09-14T07:37:43.399 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66755.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66656.log 2024-09-14T07:37:43.399 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75603.log 2024-09-14T07:37:43.400 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133162.log 2024-09-14T07:37:43.400 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.94275.log.gz 2024-09-14T07:37:43.405 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.66656.log: /var/log/ceph/cluster2-client.mirror.75603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68476.log 2024-09-14T07:37:43.405 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124277.log 2024-09-14T07:37:43.405 INFO:teuthology.orchestra.run.smithi169.stderr: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133162.log.gz 2024-09-14T07:37:43.405 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131351.log 2024-09-14T07:37:43.405 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49784.log 2024-09-14T07:37:43.406 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131351.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50435.log 2024-09-14T07:37:43.406 INFO:teuthology.orchestra.run.smithi169.stderr: 87.5%/var/log/ceph/cluster2-client.mirror.49784.log: -- replaced with /var/log/ceph/cluster1-client.mirror.131351.log.gz 2024-09-14T07:37:43.406 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50156.log 2024-09-14T07:37:43.408 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82392.log 2024-09-14T07:37:43.408 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.124277.log.gz 2024-09-14T07:37:43.408 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114341.log 2024-09-14T07:37:43.408 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50156.log: /var/log/ceph/cluster2-client.mirror.82392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125848.log 2024-09-14T07:37:43.408 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114341.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.90567.log 2024-09-14T07:37:43.408 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114341.log.gz 2024-09-14T07:37:43.409 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125848.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125848.log.gz 2024-09-14T07:37:43.410 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60352.log 2024-09-14T07:37:43.410 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.0.90567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59649.log 2024-09-14T07:37:43.411 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.60352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43003.log 2024-09-14T07:37:43.411 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59649.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36442.log 2024-09-14T07:37:43.411 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39611.log 2024-09-14T07:37:43.412 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78954.log 2024-09-14T07:37:43.413 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36442.log: /var/log/ceph/cluster2-client.mirror.39611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46711.log 2024-09-14T07:37:43.413 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118613.log 2024-09-14T07:37:43.414 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46711.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55882.log.gz 2024-09-14T07:37:43.414 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102962.log 2024-09-14T07:37:43.414 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.118613.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.118613.log.gz 2024-09-14T07:37:43.415 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.116919.log 2024-09-14T07:37:43.415 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.102962.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68738.log.gz 2024-09-14T07:37:43.416 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118413.log 2024-09-14T07:37:43.417 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50435.log.gz 2024-09-14T07:37:43.417 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.50493.log.gz 2024-09-14T07:37:43.417 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.116919.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65598.log.gz 2024-09-14T07:37:43.417 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.102962.log.gz 2024-09-14T07:37:43.418 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57428.log 2024-09-14T07:37:43.418 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118413.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118413.log.gz 2024-09-14T07:37:43.421 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63390.log 2024-09-14T07:37:43.421 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57428.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75975.log 2024-09-14T07:37:43.422 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69239.log 2024-09-14T07:37:43.423 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75975.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125190.log 2024-09-14T07:37:43.427 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69239.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59288.log.gz 2024-09-14T07:37:43.427 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62367.log.gz 2024-09-14T07:37:43.427 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71072.log.gz 2024-09-14T07:37:43.427 INFO:teuthology.orchestra.run.smithi169.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.116919.log.gz 2024-09-14T07:37:43.428 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54952.log 2024-09-14T07:37:43.428 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120171.log 2024-09-14T07:37:43.428 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125190.log: /var/log/ceph/cluster2-client.mirror.54952.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123528.log 2024-09-14T07:37:43.428 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120171.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72188.log.gz 2024-09-14T07:37:43.429 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125190.log.gz 2024-09-14T07:37:43.429 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39855.log 2024-09-14T07:37:43.430 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123528.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123528.log.gz 2024-09-14T07:37:43.431 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90797.log 2024-09-14T07:37:43.431 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54952.log.gz 2024-09-14T07:37:43.431 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55289.log.gz 2024-09-14T07:37:43.432 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39855.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.47548.log.gz 2024-09-14T07:37:43.432 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60259.log.gz 2024-09-14T07:37:43.432 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118286.log 2024-09-14T07:37:43.433 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90797.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90797.log.gz 2024-09-14T07:37:43.433 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61623.log 2024-09-14T07:37:43.434 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118286.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118286.log.gz 2024-09-14T07:37:43.434 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40953.log 2024-09-14T07:37:43.435 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107054.log 2024-09-14T07:37:43.436 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44735.log 2024-09-14T07:37:43.437 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124835.log 2024-09-14T07:37:43.437 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107054.log.gz 2024-09-14T07:37:43.437 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90318.log 2024-09-14T07:37:43.438 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124835.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.124835.log.gz 2024-09-14T07:37:43.439 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71386.log 2024-09-14T07:37:43.439 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90318.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90318.log.gz 2024-09-14T07:37:43.440 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36128.log 2024-09-14T07:37:43.440 INFO:teuthology.orchestra.run.smithi169.stderr: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120171.log.gz 2024-09-14T07:37:43.440 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71386.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34767.log.gz 2024-09-14T07:37:43.440 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37244.log 2024-09-14T07:37:43.441 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36128.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124206.log 2024-09-14T07:37:43.441 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70328.log 2024-09-14T07:37:43.441 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39297.log.gz 2024-09-14T07:37:43.442 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81013.log 2024-09-14T07:37:43.443 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70328.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61282.log 2024-09-14T07:37:43.443 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81013.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82326.log 2024-09-14T07:37:43.444 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126633.log 2024-09-14T07:37:43.444 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124206.log.gz 2024-09-14T07:37:43.444 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70235.log 2024-09-14T07:37:43.446 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112675.log 2024-09-14T07:37:43.446 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126633.log: /var/log/ceph/cluster2-client.mirror.70235.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132307.log 2024-09-14T07:37:43.446 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126633.log.gz 2024-09-14T07:37:43.446 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112675.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112675.log.gz 2024-09-14T07:37:43.446 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39669.log 2024-09-14T07:37:43.447 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51830.log 2024-09-14T07:37:43.447 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132307.log: /var/log/ceph/cluster2-client.mirror.39669.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132307.log.gz 2024-09-14T07:37:43.448 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119341.log 2024-09-14T07:37:43.448 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39855.log.gz 2024-09-14T07:37:43.448 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40586.log 2024-09-14T07:37:43.449 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110565.log 2024-09-14T07:37:43.450 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78282.log.gz 2024-09-14T07:37:43.450 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78247.log.gz 2024-09-14T07:37:43.450 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40586.log: 92.2% 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64599.log.gz 2024-09-14T07:37:43.450 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.119341.log.gz 2024-09-14T07:37:43.450 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36442.log.gz 2024-09-14T07:37:43.450 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59009.log 2024-09-14T07:37:43.452 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109749.log 2024-09-14T07:37:43.452 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110565.log: /var/log/ceph/cluster2-client.mirror.59009.log: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67034.log 2024-09-14T07:37:43.452 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.110565.log.gz 2024-09-14T07:37:43.452 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109749.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109749.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43840.log 2024-09-14T07:37:43.452 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:43.453 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132946.log 2024-09-14T07:37:43.455 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67034.log: /var/log/ceph/cluster2-client.mirror.43840.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88639.log.gz 2024-09-14T07:37:43.456 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100229.log 2024-09-14T07:37:43.456 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79696.log 2024-09-14T07:37:43.457 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132946.log: /var/log/ceph/cluster2-client.mirror.100229.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100229.log.gz 2024-09-14T07:37:43.457 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125631.log 2024-09-14T07:37:43.458 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57521.log 2024-09-14T07:37:43.460 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83604.log 2024-09-14T07:37:43.460 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.125631.log.gz 2024-09-14T07:37:43.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45221.log 2024-09-14T07:37:43.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83604.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35512.log 2024-09-14T07:37:43.461 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115920.log 2024-09-14T07:37:43.462 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128573.log 2024-09-14T07:37:43.462 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121390.log 2024-09-14T07:37:43.463 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115920.log.gz 2024-09-14T07:37:43.464 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47978.log.gz 2024-09-14T07:37:43.464 INFO:teuthology.orchestra.run.smithi169.stderr: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130912.log.gz 2024-09-14T07:37:43.464 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121390.log: /var/log/ceph/cluster1-client.mirror.128573.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121390.log.gz 2024-09-14T07:37:43.464 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128573.log.gz 2024-09-14T07:37:43.465 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84816.log 2024-09-14T07:37:43.465 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52616.log 2024-09-14T07:37:43.466 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84816.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129023.log 2024-09-14T07:37:43.467 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47083.log 2024-09-14T07:37:43.467 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129023.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41511.log 2024-09-14T07:37:43.467 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129023.log.gz 2024-09-14T07:37:43.467 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73581.log 2024-09-14T07:37:43.467 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77874.log 2024-09-14T07:37:43.467 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41511.log: /var/log/ceph/cluster2-client.mirror.73581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88286.log 2024-09-14T07:37:43.469 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46990.log 2024-09-14T07:37:43.469 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89372.log 2024-09-14T07:37:43.469 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.88286.log: /var/log/ceph/cluster2-client.mirror.46990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119623.log 2024-09-14T07:37:43.469 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79964.log 2024-09-14T07:37:43.470 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119623.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.119623.log.gz 2024-09-14T07:37:43.470 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125423.log 2024-09-14T07:37:43.470 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132169.log 2024-09-14T07:37:43.471 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125423.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88286.log.gz 2024-09-14T07:37:43.472 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63576.log 2024-09-14T07:37:43.472 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.125423.log.gz 2024-09-14T07:37:43.472 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74922.log 2024-09-14T07:37:43.473 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132169.log: /var/log/ceph/cluster2-client.mirror.63576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134275.log 2024-09-14T07:37:43.473 INFO:teuthology.orchestra.run.smithi169.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.132169.log.gz 2024-09-14T07:37:43.473 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68113.log 2024-09-14T07:37:43.474 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123122.log 2024-09-14T07:37:43.474 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134275.log: /var/log/ceph/cluster2-client.mirror.68113.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134275.log.gz 2024-09-14T07:37:43.475 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100831.log 2024-09-14T07:37:43.475 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35849.log 2024-09-14T07:37:43.475 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123122.log.gz 2024-09-14T07:37:43.476 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121467.log 2024-09-14T07:37:43.476 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35849.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100831.log.gz 2024-09-14T07:37:43.476 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52683.log 2024-09-14T07:37:43.476 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64413.log 2024-09-14T07:37:43.476 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.121467.log.gz 2024-09-14T07:37:43.477 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.52683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85465.log 2024-09-14T07:37:43.477 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64413.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100916.log 2024-09-14T07:37:43.478 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29927.log 2024-09-14T07:37:43.478 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85465.log: /var/log/ceph/cluster2-client.mirror.100916.log: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115150.log 2024-09-14T07:37:43.478 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.100916.log.gz 2024-09-14T07:37:43.479 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113550.log 2024-09-14T07:37:43.480 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29927.log: /var/log/ceph/cluster2-client.mirror.115150.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29927.log.gz 2024-09-14T07:37:43.480 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70235.log.gz 2024-09-14T07:37:43.480 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115150.log.gz 2024-09-14T07:37:43.480 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66755.log.gz 2024-09-14T07:37:43.480 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113148.log 2024-09-14T07:37:43.481 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113550.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95246.log 2024-09-14T07:37:43.481 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.113550.log.gz 2024-09-14T07:37:43.482 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73302.log.gz 2024-09-14T07:37:43.482 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127403.log 2024-09-14T07:37:43.482 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.113148.log: /var/log/ceph/cluster2-client.mirror.95246.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95246.log.gz 2024-09-14T07:37:43.482 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113148.log.gz 2024-09-14T07:37:43.482 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62274.log 2024-09-14T07:37:43.482 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82124.log.gz 2024-09-14T07:37:43.483 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127403.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127403.log.gz 2024-09-14T07:37:43.483 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65429.log 2024-09-14T07:37:43.483 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82998.log 2024-09-14T07:37:43.483 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.65429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41476.log 2024-09-14T07:37:43.484 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41325.log 2024-09-14T07:37:43.485 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85641.log 2024-09-14T07:37:43.485 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41325.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30330.log 2024-09-14T07:37:43.486 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85641.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131277.log 2024-09-14T07:37:43.486 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30330.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30330.log.gz 2024-09-14T07:37:43.486 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128078.log 2024-09-14T07:37:43.487 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131277.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.131277.log.gz 2024-09-14T07:37:43.487 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30862.log 2024-09-14T07:37:43.488 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128078.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128078.log.gz 2024-09-14T07:37:43.489 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46525.log 2024-09-14T07:37:43.490 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35884.log 2024-09-14T07:37:43.490 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.30862.log: /var/log/ceph/cluster2-client.mirror.46525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70142.log 2024-09-14T07:37:43.491 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106973.log 2024-09-14T07:37:43.492 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47327.log 2024-09-14T07:37:43.492 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106973.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106973.log.gz 2024-09-14T07:37:43.493 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.132682.log 2024-09-14T07:37:43.493 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45552.log 2024-09-14T07:37:43.496 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.2.132682.log: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.88906.log.gz 2024-09-14T07:37:43.496 INFO:teuthology.orchestra.run.smithi169.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.132682.log.gz 2024-09-14T07:37:43.496 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39704.log 2024-09-14T07:37:43.497 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45552.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129225.log 2024-09-14T07:37:43.497 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128258.log 2024-09-14T07:37:43.498 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93401.log 2024-09-14T07:37:43.499 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129225.log.gz 2024-09-14T07:37:43.499 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128258.log: 86.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110776.log 2024-09-14T07:37:43.499 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.128258.log.gz 2024-09-14T07:37:43.499 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51086.log 2024-09-14T07:37:43.499 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129483.log 2024-09-14T07:37:43.503 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110776.log: /var/log/ceph/cluster2-client.mirror.51086.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.30430.log.gz 2024-09-14T07:37:43.503 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126704.log 2024-09-14T07:37:43.504 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.129483.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.129483.log.gz 2024-09-14T07:37:43.505 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105338.log 2024-09-14T07:37:43.505 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126704.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126704.log.gz 2024-09-14T07:37:43.506 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45967.log 2024-09-14T07:37:43.506 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105338.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105338.log.gz 2024-09-14T07:37:43.507 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76386.log 2024-09-14T07:37:43.508 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51144.log 2024-09-14T07:37:43.508 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76386.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106561.log 2024-09-14T07:37:43.509 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34453.log 2024-09-14T07:37:43.510 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106561.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106561.log.gz 2024-09-14T07:37:43.510 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75824.log 2024-09-14T07:37:43.511 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34453.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93132.log 2024-09-14T07:37:43.512 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56161.log 2024-09-14T07:37:43.512 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93132.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93132.log.gz 2024-09-14T07:37:43.514 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106526.log 2024-09-14T07:37:43.514 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75180.log 2024-09-14T07:37:43.515 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106526.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106526.log.gz 2024-09-14T07:37:43.516 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89933.log 2024-09-14T07:37:43.520 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122167.log 2024-09-14T07:37:43.524 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89933.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69154.log.gz 2024-09-14T07:37:43.531 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79595.log.gz 2024-09-14T07:37:43.534 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40732.log.gz 2024-09-14T07:37:43.548 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132221.log 2024-09-14T07:37:43.555 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76816.log.gz 2024-09-14T07:37:43.555 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50586.log.gz 2024-09-14T07:37:43.556 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122167.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122167.log.gz 2024-09-14T07:37:43.556 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55568.log 2024-09-14T07:37:43.557 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132221.log: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132221.log.gz 2024-09-14T07:37:43.565 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55603.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.55661.log.gz 2024-09-14T07:37:43.565 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:43.566 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94453.log 2024-09-14T07:37:43.566 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59009.log.gz 2024-09-14T07:37:43.566 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62088.log.gz 2024-09-14T07:37:43.566 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55568.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40953.log.gz 2024-09-14T07:37:43.567 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61623.log.gz 2024-09-14T07:37:43.568 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68055.log.gz 2024-09-14T07:37:43.568 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63576.log.gz 2024-09-14T07:37:43.576 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52756.log 2024-09-14T07:37:43.577 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94453.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94453.log.gz 2024-09-14T07:37:43.579 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45967.log.gz 2024-09-14T07:37:43.587 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69097.log.gz 2024-09-14T07:37:43.588 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35977.log.gz 2024-09-14T07:37:43.591 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37244.log.gz 2024-09-14T07:37:43.594 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43526.log.gz 2024-09-14T07:37:43.594 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44549.log.gz 2024-09-14T07:37:43.596 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45016.log.gz 2024-09-14T07:37:43.596 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51830.log.gz 2024-09-14T07:37:43.596 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57521.log.gz 2024-09-14T07:37:43.600 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40414.log 2024-09-14T07:37:43.600 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.52756.log: -- replaced with /var/log/ceph/cluster2-client.mirror.41325.log.gz 2024-09-14T07:37:43.604 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70142.log.gz 2024-09-14T07:37:43.604 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75638.log.gz 2024-09-14T07:37:43.605 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75731.log.gz 2024-09-14T07:37:43.606 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76386.log.gz 2024-09-14T07:37:43.624 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51737.log 2024-09-14T07:37:43.629 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.40414.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82897.log.gz 2024-09-14T07:37:43.632 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78888.log.gz 2024-09-14T07:37:43.636 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131236.log 2024-09-14T07:37:43.656 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51737.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40151.log 2024-09-14T07:37:43.657 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131236.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.131236.log.gz 2024-09-14T07:37:43.668 INFO:teuthology.orchestra.run.smithi169.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85809.log.gz 2024-09-14T07:37:43.672 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74546.log 2024-09-14T07:37:43.676 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64262.log 2024-09-14T07:37:43.677 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74546.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110523.log 2024-09-14T07:37:43.691 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64262.log: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.131612.log.gz 2024-09-14T07:37:43.691 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30982.log 2024-09-14T07:37:43.693 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110523.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110523.log.gz 2024-09-14T07:37:43.693 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42259.log 2024-09-14T07:37:43.694 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51888.log 2024-09-14T07:37:43.695 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42259.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132946.log.gz 2024-09-14T07:37:43.695 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126511.log 2024-09-14T07:37:43.696 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7%/var/log/ceph/cluster2-client.mirror.51888.log: -- replaced with /var/log/ceph/cluster2-client.mirror.89372.log.gz 2024-09-14T07:37:43.696 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92344.log 2024-09-14T07:37:43.697 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98343.log 2024-09-14T07:37:43.698 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126511.log: /var/log/ceph/cluster2-client.mirror.92344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55103.log 2024-09-14T07:37:43.698 INFO:teuthology.orchestra.run.smithi169.stderr: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92344.log.gz 2024-09-14T07:37:43.698 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44026.log 2024-09-14T07:37:43.698 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.98343.log.gz 2024-09-14T07:37:43.698 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.55103.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77374.log 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57835.log 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44026.log: /var/log/ceph/cluster2-client.mirror.77374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52651.log 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126511.log.gz 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120869.log 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57835.log: /var/log/ceph/cluster2-client.mirror.52651.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84248.log 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132247.log 2024-09-14T07:37:43.700 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120869.log: /var/log/ceph/cluster2-client.mirror.84248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74697.log 2024-09-14T07:37:43.702 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.132247.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.120869.log.gz 2024-09-14T07:37:43.702 INFO:teuthology.orchestra.run.smithi169.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.132247.log.gz 2024-09-14T07:37:43.702 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37000.log 2024-09-14T07:37:43.702 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58393.log 2024-09-14T07:37:43.703 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74697.log: /var/log/ceph/cluster2-client.mirror.37000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128663.log 2024-09-14T07:37:43.703 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.58393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95204.log 2024-09-14T07:37:43.703 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95724.log 2024-09-14T07:37:43.705 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68908.log 2024-09-14T07:37:43.706 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128663.log: /var/log/ceph/cluster2-client.mirror.95204.log: /var/log/ceph/cluster2-client.mirror.95724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67743.log 2024-09-14T07:37:43.706 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128663.log.gz 2024-09-14T07:37:43.706 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68908.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95724.log.gz 2024-09-14T07:37:43.706 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54022.log.gz 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127740.log 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95204.log.gz 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67743.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55568.log.gz 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50714.log.gz 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54824.log 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104608.log 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127740.log: /var/log/ceph/cluster2-client.mirror.54824.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127740.log.gz 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59783.log.gz 2024-09-14T07:37:43.708 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49129.log 2024-09-14T07:37:43.710 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95400.log 2024-09-14T07:37:43.710 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104608.log: /var/log/ceph/cluster2-client.mirror.49129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122760.log 2024-09-14T07:37:43.710 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59137.log.gz 2024-09-14T07:37:43.710 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36035.log 2024-09-14T07:37:43.710 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122760.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95400.log.gz 2024-09-14T07:37:43.710 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122760.log.gz 2024-09-14T07:37:43.711 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81922.log.gz 2024-09-14T07:37:43.711 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.104608.log.gz 2024-09-14T07:37:43.711 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82225.log 2024-09-14T07:37:43.712 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111654.log 2024-09-14T07:37:43.712 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60352.log.gz 2024-09-14T07:37:43.712 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36035.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63390.log.gz 2024-09-14T07:37:43.712 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31067.log 2024-09-14T07:37:43.712 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.82225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126031.log 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.111654.log: /var/log/ceph/cluster2-client.mirror.31067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38642.log 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126031.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59649.log.gz 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115352.log 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.111654.log.gz 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80912.log 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.126031.log.gz 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130714.log 2024-09-14T07:37:43.714 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115352.log: /var/log/ceph/cluster2-client.mirror.80912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87575.log 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127538.log 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115352.log.gz 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87575.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130714.log.gz 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45298.log 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr:gzip 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85488.log.gz -5 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.80166.log 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127538.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40151.log.gz/var/log/ceph/cluster2-client.mirror.45298.log: 2024-09-14T07:37:43.716 INFO:teuthology.orchestra.run.smithi169.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127538.log.gz 2024-09-14T07:37:43.717 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69863.log 2024-09-14T07:37:43.717 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119316.log 2024-09-14T07:37:43.720 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80166.log: /var/log/ceph/cluster2-client.mirror.69863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117916.log 2024-09-14T07:37:43.721 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84156.log 2024-09-14T07:37:43.721 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.119316.log: /var/log/ceph/cluster2-client.mirror.117916.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119919.log 2024-09-14T07:37:43.721 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84156.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.117916.log.gz 2024-09-14T07:37:43.721 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.119316.log.gz 2024-09-14T07:37:43.722 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126775.log 2024-09-14T07:37:43.722 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56684.log 2024-09-14T07:37:43.722 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49226.log 2024-09-14T07:37:43.723 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126775.log.gz 2024-09-14T07:37:43.723 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56684.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50214.log 2024-09-14T07:37:43.724 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103803.log 2024-09-14T07:37:43.724 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49226.log: /var/log/ceph/cluster2-client.mirror.50214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133026.log 2024-09-14T07:37:43.724 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84415.log 2024-09-14T07:37:43.726 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103803.log: /var/log/ceph/cluster2-client.mirror.133026.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133026.log.gz 2024-09-14T07:37:43.726 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.103803.log.gz 2024-09-14T07:37:43.727 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121178.log 2024-09-14T07:37:43.727 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64506.log 2024-09-14T07:37:43.727 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84415.log: /var/log/ceph/cluster2-client.mirror.121178.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.121178.log.gz 2024-09-14T07:37:43.730 INFO:teuthology.orchestra.run.smithi169.stderr: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89933.log.gz 2024-09-14T07:37:43.730 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61658.log 2024-09-14T07:37:43.731 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122581.log 2024-09-14T07:37:43.731 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64506.log: /var/log/ceph/cluster2-client.mirror.61658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115839.log 2024-09-14T07:37:43.731 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84730.log 2024-09-14T07:37:43.732 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122581.log: /var/log/ceph/cluster2-client.mirror.115839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35791.log 2024-09-14T07:37:43.732 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122581.log.gz 2024-09-14T07:37:43.732 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.115839.log.gz 2024-09-14T07:37:43.732 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.84730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133629.log 2024-09-14T07:37:43.732 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43898.log 2024-09-14T07:37:43.734 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35791.log: /var/log/ceph/cluster1-client.mirror.133629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133987.log 2024-09-14T07:37:43.734 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47606.log 2024-09-14T07:37:43.734 INFO:teuthology.orchestra.run.smithi169.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.133629.log.gz 2024-09-14T07:37:43.734 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108580.log 2024-09-14T07:37:43.734 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133987.log: /var/log/ceph/cluster2-client.mirror.47606.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125261.log 2024-09-14T07:37:43.734 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.133987.log.gz 2024-09-14T07:37:43.735 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108580.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36221.log 2024-09-14T07:37:43.735 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125261.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.108580.log.gz 2024-09-14T07:37:43.735 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97434.log 2024-09-14T07:37:43.736 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44177.log 2024-09-14T07:37:43.736 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36221.log: /var/log/ceph/cluster2-client.mirror.97434.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125261.log.gz 2024-09-14T07:37:43.736 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75696.log 2024-09-14T07:37:43.736 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97434.log.gz 2024-09-14T07:37:43.736 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103711.log 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75696.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129180.log 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36035.log.gz 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76010.log/var/log/ceph/cluster2-client.mirror.103711.log: 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36128.log.gz 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36221.log.gz 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103711.log.gz 2024-09-14T07:37:43.738 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129180.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41794.log.gz 2024-09-14T07:37:43.739 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42259.log.gz 2024-09-14T07:37:43.739 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43189.log.gz 2024-09-14T07:37:43.739 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43003.log.gz 2024-09-14T07:37:43.739 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129180.log.gz 2024-09-14T07:37:43.740 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80912.log.gz 2024-09-14T07:37:43.740 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120149.log 2024-09-14T07:37:43.741 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70328.log.gz 2024-09-14T07:37:43.741 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51179.log 2024-09-14T07:37:43.741 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82225.log.gz 2024-09-14T07:37:43.741 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82326.log.gz 2024-09-14T07:37:43.741 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.120149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75276.log 2024-09-14T07:37:43.741 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66656.log.gz 2024-09-14T07:37:43.742 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69619.log 2024-09-14T07:37:43.742 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67034.log.gz 2024-09-14T07:37:43.742 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.75276.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120149.log.gz 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67743.log.gz 2024-09-14T07:37:43.743 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:43.743 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47641.log 2024-09-14T07:37:43.743 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75696.log.gz 2024-09-14T07:37:43.744 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70700.log 2024-09-14T07:37:43.744 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69619.log: /var/log/ceph/cluster2-client.mirror.47641.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74325.log.gz 2024-09-14T07:37:43.744 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76010.log.gz 2024-09-14T07:37:43.744 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.72865.log 2024-09-14T07:37:43.745 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113407.log 2024-09-14T07:37:43.745 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.1.72865.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73581.log.gz 2024-09-14T07:37:43.745 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126611.log 2024-09-14T07:37:43.745 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77839.log 2024-09-14T07:37:43.745 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113407.log.gz 2024-09-14T07:37:43.746 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132328.log 2024-09-14T07:37:43.747 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77037.log 2024-09-14T07:37:43.747 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77839.log: /var/log/ceph/cluster2-client.mirror.132328.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126611.log.gz 2024-09-14T07:37:43.747 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53935.log 2024-09-14T07:37:43.747 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132328.log.gz 2024-09-14T07:37:43.748 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43898.log.gz 2024-09-14T07:37:43.748 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77037.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87575.log.gz 2024-09-14T07:37:43.748 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.43840.log.gz 2024-09-14T07:37:43.748 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54673.log 2024-09-14T07:37:43.749 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96939.log 2024-09-14T07:37:43.749 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64692.log 2024-09-14T07:37:43.749 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56533.log 2024-09-14T07:37:43.749 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96939.log: /var/log/ceph/cluster2-client.mirror.64692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42968.log 2024-09-14T07:37:43.749 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56533.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96939.log.gz 2024-09-14T07:37:43.752 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116653.log 2024-09-14T07:37:43.752 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42968.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133602.log 2024-09-14T07:37:43.754 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128235.log 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116653.log: /var/log/ceph/cluster1-client.mirror.133602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30078.log 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128235.log: 87.5% 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.116653.log.gz 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.133602.log.gz 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128235.log.gz 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88935.log 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30078.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76944.log 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30078.log.gz 2024-09-14T07:37:43.755 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.88935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114306.log 2024-09-14T07:37:43.756 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68866.log 2024-09-14T07:37:43.756 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76944.log: /var/log/ceph/cluster2-client.mirror.114306.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115227.log 2024-09-14T07:37:43.756 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.114306.log.gz 2024-09-14T07:37:43.756 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56812.log 2024-09-14T07:37:43.757 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52016.log 2024-09-14T07:37:43.757 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115227.log.gz 2024-09-14T07:37:43.758 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56812.log: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88935.log.gz 2024-09-14T07:37:43.760 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125161.log 2024-09-14T07:37:43.760 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108936.log 2024-09-14T07:37:43.761 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52016.log: /var/log/ceph/cluster2-client.mirror.125161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32310.log 2024-09-14T07:37:43.762 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.108936.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91979.log 2024-09-14T07:37:43.762 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32310.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.108936.log.gz 2024-09-14T07:37:43.762 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91979.log: 3.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91979.log.gz 2024-09-14T07:37:43.764 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63797.log 2024-09-14T07:37:43.764 INFO:teuthology.orchestra.run.smithi169.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125161.log.gz 2024-09-14T07:37:43.764 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.119406.log 2024-09-14T07:37:43.765 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91114.log 2024-09-14T07:37:43.765 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.119406.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30480.log 2024-09-14T07:37:43.766 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68620.log 2024-09-14T07:37:43.773 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91114.log: /var/log/ceph/cluster2-client.mirror.30480.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85078.log 2024-09-14T07:37:43.774 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.68620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108638.log 2024-09-14T07:37:43.775 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116770.log 2024-09-14T07:37:43.775 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108638.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108638.log.gz 2024-09-14T07:37:43.776 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108451.log 2024-09-14T07:37:43.776 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.110776.log.gz 2024-09-14T07:37:43.777 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.116770.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81079.log 88.1% 2024-09-14T07:37:43.777 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.116770.log.gz 2024-09-14T07:37:43.778 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.108451.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.119406.log.gz 2024-09-14T07:37:43.792 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120574.log 2024-09-14T07:37:43.792 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52481.log 2024-09-14T07:37:43.793 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120574.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120574.log.gz 2024-09-14T07:37:43.793 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52109.log 2024-09-14T07:37:43.794 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103330.log 2024-09-14T07:37:43.794 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133260.log 2024-09-14T07:37:43.795 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103330.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.103330.log.gz 2024-09-14T07:37:43.796 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107531.log 2024-09-14T07:37:43.796 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.133260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48478.log 2024-09-14T07:37:43.797 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.107531.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.133260.log.gz 2024-09-14T07:37:43.797 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107531.log.gz 2024-09-14T07:37:43.798 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45529.log.gz 2024-09-14T07:37:43.798 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128145.log 2024-09-14T07:37:43.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91386.log 2024-09-14T07:37:43.799 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78585.log 2024-09-14T07:37:43.800 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8%/var/log/ceph/cluster2-client.mirror.91386.log: -- replaced with /var/log/ceph/cluster1-client.mirror.128145.log.gz 2024-09-14T07:37:43.800 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91386.log.gz 2024-09-14T07:37:43.800 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84839.log 2024-09-14T07:37:43.800 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101346.log 2024-09-14T07:37:43.801 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126298.log 2024-09-14T07:37:43.801 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110677.log 2024-09-14T07:37:43.801 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126298.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101346.log.gz 2024-09-14T07:37:43.802 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57149.log 2024-09-14T07:37:43.803 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110677.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.126298.log.gz 2024-09-14T07:37:43.804 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68866.log.gz 2024-09-14T07:37:43.804 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68908.log.gz 2024-09-14T07:37:43.804 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110677.log.gz 2024-09-14T07:37:43.804 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65970.log 2024-09-14T07:37:43.805 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.57149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38398.log 2024-09-14T07:37:43.805 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65970.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130594.log 2024-09-14T07:37:43.806 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38398.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.99333.log 2024-09-14T07:37:43.806 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74825.log 2024-09-14T07:37:43.806 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.99333.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.130594.log.gz 2024-09-14T07:37:43.808 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113173.log 2024-09-14T07:37:43.809 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.74825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51330.log 2024-09-14T07:37:43.810 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113173.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.113173.log.gz 2024-09-14T07:37:43.811 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124103.log 2024-09-14T07:37:43.811 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51330.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103245.log 2024-09-14T07:37:43.812 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124103.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124103.log.gz 2024-09-14T07:37:43.813 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123087.log 2024-09-14T07:37:43.813 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103245.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103245.log.gz 2024-09-14T07:37:43.814 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51423.log 2024-09-14T07:37:43.814 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123087.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123087.log.gz 2024-09-14T07:37:43.815 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76444.log 2024-09-14T07:37:43.816 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.51423.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49129.log.gz 2024-09-14T07:37:43.816 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65877.log.gz 2024-09-14T07:37:43.817 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62274.log.gz 2024-09-14T07:37:43.820 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50772.log 2024-09-14T07:37:43.821 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102427.log 2024-09-14T07:37:43.821 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102808.log 2024-09-14T07:37:43.822 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102427.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102427.log.gz 2024-09-14T07:37:43.822 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131195.log 2024-09-14T07:37:43.823 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102808.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102808.log.gz 2024-09-14T07:37:43.823 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129360.log 2024-09-14T07:37:43.824 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131195.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.131195.log.gz 2024-09-14T07:37:43.825 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73337.log 2024-09-14T07:37:43.825 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129360.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.129360.log.gz 2024-09-14T07:37:43.826 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40504.log 2024-09-14T07:37:43.826 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128438.log 2024-09-14T07:37:43.827 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72653.log 2024-09-14T07:37:43.827 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128438.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128438.log.gz 2024-09-14T07:37:43.829 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47641.log.gz 2024-09-14T07:37:43.830 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102715.log 2024-09-14T07:37:43.831 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125070.log 2024-09-14T07:37:43.831 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102715.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102715.log.gz 2024-09-14T07:37:43.832 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73244.log 2024-09-14T07:37:43.832 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125070.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125070.log.gz 2024-09-14T07:37:43.833 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69129.log 2024-09-14T07:37:43.833 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110949.log 2024-09-14T07:37:43.834 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.69129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116885.log 2024-09-14T07:37:43.835 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110949.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.110949.log.gz 2024-09-14T07:37:43.835 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68206.log 2024-09-14T07:37:43.836 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.116885.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.119919.log.gz 2024-09-14T07:37:43.837 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49563.log 2024-09-14T07:37:43.837 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.68206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29766.log 2024-09-14T07:37:43.838 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49563.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100296.log 2024-09-14T07:37:43.838 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29766.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29766.log.gz 2024-09-14T07:37:43.839 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104628.log 2024-09-14T07:37:43.839 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.100296.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.100296.log.gz 2024-09-14T07:37:43.840 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86410.log 2024-09-14T07:37:43.841 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.104628.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.99333.log.gz 2024-09-14T07:37:43.843 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112197.log 2024-09-14T07:37:43.843 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86410.log: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.104628.log.gz 2024-09-14T07:37:43.843 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86816.log 2024-09-14T07:37:43.843 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112197.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48013.log 2024-09-14T07:37:43.843 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86816.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.112197.log.gz 2024-09-14T07:37:43.844 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97121.log 2024-09-14T07:37:43.844 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48013.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98901.log 2024-09-14T07:37:43.844 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118251.log 2024-09-14T07:37:43.846 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98901.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97121.log.gz 2024-09-14T07:37:43.848 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125469.log 2024-09-14T07:37:43.849 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118251.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118251.log.gz 2024-09-14T07:37:43.849 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85555.log 2024-09-14T07:37:43.850 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125469.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.125469.log.gz 2024-09-14T07:37:43.850 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62030.log 2024-09-14T07:37:43.850 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35512.log.gz 2024-09-14T07:37:43.851 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61530.log 2024-09-14T07:37:43.851 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38642.log.gz 2024-09-14T07:37:43.852 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.134008.log 2024-09-14T07:37:43.852 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61530.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52683.log.gz 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64913.log.gz 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81079.log.gz 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35884.log.gz 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41511.log.gz 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71537.log 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42968.log.gz 2024-09-14T07:37:43.854 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.134008.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.134008.log.gz 2024-09-14T07:37:43.855 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80407.log 2024-09-14T07:37:43.855 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50958.log 2024-09-14T07:37:43.855 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44456.log 2024-09-14T07:37:43.856 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.50958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45051.log 2024-09-14T07:37:43.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.44456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81786.log 2024-09-14T07:37:43.857 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47234.log 2024-09-14T07:37:43.858 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81786.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56812.log.gz 2024-09-14T07:37:43.858 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37747.log 2024-09-14T07:37:43.858 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56998.log.gz 2024-09-14T07:37:43.859 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47234.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57149.log.gz 2024-09-14T07:37:43.860 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65633.log 2024-09-14T07:37:43.860 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37747.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86650.log 2024-09-14T07:37:43.862 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65633.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70700.log.gz 2024-09-14T07:37:43.864 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64785.log 2024-09-14T07:37:43.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32775.log 2024-09-14T07:37:43.865 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.64785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107697.log 2024-09-14T07:37:43.866 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.32775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90432.log 2024-09-14T07:37:43.867 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.107697.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93401.log.gz 2024-09-14T07:37:43.867 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69619.log.gz 2024-09-14T07:37:43.868 INFO:teuthology.orchestra.run.smithi169.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.107697.log.gz 2024-09-14T07:37:43.869 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89999.log 2024-09-14T07:37:43.869 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.90432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132409.log 2024-09-14T07:37:43.869 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89999.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89999.log.gz 2024-09-14T07:37:43.884 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43805.log 2024-09-14T07:37:43.884 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80407.log.gz 2024-09-14T07:37:43.884 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132409.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.132409.log.gz 2024-09-14T07:37:43.885 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93217.log 2024-09-14T07:37:43.885 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98152.log 2024-09-14T07:37:43.886 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93217.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93217.log.gz 2024-09-14T07:37:43.886 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110821.log 2024-09-14T07:37:43.887 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98152.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69174.log.gz 2024-09-14T07:37:43.888 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.98152.log.gz 2024-09-14T07:37:43.889 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113585.log 2024-09-14T07:37:43.889 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110821.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110821.log.gz 2024-09-14T07:37:43.890 INFO:teuthology.orchestra.run.smithi169.stderr: 95.1% -- replaced with /var/log/ceph/cluster2-client.mirror.91114.log.gz 2024-09-14T07:37:43.890 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41011.log 2024-09-14T07:37:43.891 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101245.log 2024-09-14T07:37:43.891 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.41011.log: -- replaced with /var/log/ceph/cluster2-client.mirror.113585.log.gz 2024-09-14T07:37:43.891 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85058.log 2024-09-14T07:37:43.891 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81988.log 2024-09-14T07:37:43.892 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85058.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101245.log.gz 2024-09-14T07:37:43.892 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119284.log 2024-09-14T07:37:43.892 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129113.log 2024-09-14T07:37:43.893 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119284.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63762.log 2024-09-14T07:37:43.893 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129113.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119284.log.gz 2024-09-14T07:37:43.893 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.129113.log.gz 2024-09-14T07:37:43.893 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130672.log 2024-09-14T07:37:43.894 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78484.log 2024-09-14T07:37:43.894 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.130672.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100754.log 2024-09-14T07:37:43.895 INFO:teuthology.orchestra.run.smithi169.stderr: 86.7%/var/log/ceph/cluster2-client.mirror.78484.log: -- replaced with /var/log/ceph/cluster1-client.mirror.130672.log.gz 2024-09-14T07:37:43.895 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42910.log 2024-09-14T07:37:43.895 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.87132.log 2024-09-14T07:37:43.895 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42910.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.100754.log.gz 2024-09-14T07:37:43.896 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122971.log 2024-09-14T07:37:43.896 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.87132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84925.log 2024-09-14T07:37:43.897 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87598.log 2024-09-14T07:37:43.897 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.84925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79156.log 2024-09-14T07:37:43.898 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.87598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121708.log 2024-09-14T07:37:43.898 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79156.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94037.log 2024-09-14T07:37:43.901 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.121708.log: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.121708.log.gz 2024-09-14T07:37:43.901 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89131.log 2024-09-14T07:37:43.902 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94037.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94037.log.gz 2024-09-14T07:37:43.902 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126124.log 2024-09-14T07:37:43.903 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89131.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89131.log.gz 2024-09-14T07:37:43.903 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132652.log 2024-09-14T07:37:43.904 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126124.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126124.log.gz 2024-09-14T07:37:43.904 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114136.log 2024-09-14T07:37:43.906 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.132652.log: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132652.log.gz 2024-09-14T07:37:43.906 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104076.log 2024-09-14T07:37:43.907 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.114136.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114136.log.gz 2024-09-14T07:37:43.907 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130025.log 2024-09-14T07:37:43.908 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.104076.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104076.log.gz 2024-09-14T07:37:43.908 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81281.log 2024-09-14T07:37:43.910 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130025.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69197.log.gz 2024-09-14T07:37:43.910 INFO:teuthology.orchestra.run.smithi169.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130025.log.gz 2024-09-14T07:37:43.910 INFO:teuthology.orchestra.run.smithi169.stderr: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122971.log.gz 2024-09-14T07:37:43.910 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77467.log 2024-09-14T07:37:43.911 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81281.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115394.log 2024-09-14T07:37:43.911 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38770.log 2024-09-14T07:37:43.911 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115394.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88216.log 2024-09-14T07:37:43.912 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38770.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.115394.log.gz 2024-09-14T07:37:43.912 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80574.log 2024-09-14T07:37:43.912 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88216.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47141.log 2024-09-14T07:37:43.913 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88216.log.gz 2024-09-14T07:37:43.913 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25464.log 2024-09-14T07:37:43.913 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.47141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125212.log 2024-09-14T07:37:43.914 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.25464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109402.log 2024-09-14T07:37:43.914 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25464.log.gz 2024-09-14T07:37:43.914 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125212.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125212.log.gz 2024-09-14T07:37:43.915 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127268.log 2024-09-14T07:37:43.915 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.109402.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72746.log 2024-09-14T07:37:43.916 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127268.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.109402.log.gz 2024-09-14T07:37:43.916 INFO:teuthology.orchestra.run.smithi169.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127268.log.gz 2024-09-14T07:37:43.916 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99900.log 2024-09-14T07:37:43.916 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73430.log 2024-09-14T07:37:43.917 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.99900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69991.log 2024-09-14T07:37:43.918 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.73430.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99900.log.gz 2024-09-14T07:37:43.918 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121313.log 2024-09-14T07:37:43.918 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69991.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67927.log 2024-09-14T07:37:43.918 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40439.log.gz 2024-09-14T07:37:43.918 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121313.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121313.log.gz 2024-09-14T07:37:43.919 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109370.log 2024-09-14T07:37:43.919 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130693.log 2024-09-14T07:37:43.920 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69677.log 2024-09-14T07:37:43.920 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.130693.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109370.log.gz 2024-09-14T07:37:43.920 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.130693.log.gz 2024-09-14T07:37:43.921 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94337.log 2024-09-14T07:37:43.921 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.69677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120413.log 2024-09-14T07:37:43.921 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.94337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43096.log 2024-09-14T07:37:43.922 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120413.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94337.log.gz 2024-09-14T07:37:43.924 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45221.log.gz 2024-09-14T07:37:43.928 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98540.log 2024-09-14T07:37:43.928 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.43096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65540.log 2024-09-14T07:37:43.929 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.98540.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.120413.log.gz 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98540.log.gz 2024-09-14T07:37:43.929 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:43.930 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75072.log.gz 2024-09-14T07:37:43.932 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110107.log 2024-09-14T07:37:43.932 INFO:teuthology.orchestra.run.smithi169.stderr:gzip/var/log/ceph/cluster2-client.mirror.65540.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123945.log 2024-09-14T07:37:43.933 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.110107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112829.log 2024-09-14T07:37:43.933 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123945.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48815.log.gz 2024-09-14T07:37:43.935 INFO:teuthology.orchestra.run.smithi169.stderr: 92.0% 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.110107.log.gz 2024-09-14T07:37:43.935 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.123945.log.gz 2024-09-14T07:37:43.936 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85788.log 2024-09-14T07:37:43.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118332.log 2024-09-14T07:37:43.936 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85788.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.112829.log.gz 2024-09-14T07:37:43.936 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61282.log.gz 2024-09-14T07:37:43.936 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36593.log 2024-09-14T07:37:43.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2024-09-14T07:37:43.937 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36593.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118332.log.gz 2024-09-14T07:37:43.937 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100321.log 2024-09-14T07:37:43.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.130142.log 2024-09-14T07:37:43.938 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100321.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96038.log 2024-09-14T07:37:43.940 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.0.130142.log: 92.0% 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79696.log.gz 2024-09-14T07:37:43.940 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.100321.log.gz 2024-09-14T07:37:43.940 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79964.log.gz 2024-09-14T07:37:43.944 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119249.log 2024-09-14T07:37:43.945 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96038.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96038.log.gz 2024-09-14T07:37:43.945 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67185.log 2024-09-14T07:37:43.946 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.108451.log.gz 2024-09-14T07:37:43.946 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.130142.log.gz 2024-09-14T07:37:43.946 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127830.log 2024-09-14T07:37:43.947 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53391.log 2024-09-14T07:37:43.947 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67185.log: /var/log/ceph/cluster1-client.mirror.127830.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.127830.log.gz 2024-09-14T07:37:43.947 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97707.log 2024-09-14T07:37:43.948 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.119249.log: /var/log/ceph/cluster2-client.mirror.53391.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119249.log.gz 2024-09-14T07:37:43.948 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36349.log 2024-09-14T07:37:43.949 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38863.log 2024-09-14T07:37:43.949 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42782.log 2024-09-14T07:37:43.949 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97707.log.gz 2024-09-14T07:37:43.949 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36349.log: /var/log/ceph/cluster2-client.mirror.38863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49842.log 2024-09-14T07:37:43.949 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85683.log 2024-09-14T07:37:43.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.42782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45509.log 2024-09-14T07:37:43.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49842.log: /var/log/ceph/cluster2-client.mirror.85683.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36779.log 2024-09-14T07:37:43.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.45509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72281.log 2024-09-14T07:37:43.950 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.36779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111565.log 2024-09-14T07:37:43.952 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33368.log 2024-09-14T07:37:43.952 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.72281.log: /var/log/ceph/cluster2-client.mirror.111565.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62739.log.gz 2024-09-14T07:37:43.952 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.111565.log.gz 2024-09-14T07:37:43.952 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106079.log 2024-09-14T07:37:43.952 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.33368.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61003.log 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134145.log 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.106079.log: /var/log/ceph/cluster2-client.mirror.61003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48908.log 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr: 70.2% -- replaced with /var/log/ceph/cluster2-client.admin.30999.log.gz 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.31067.log.gz 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85273.log 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134145.log.gz 2024-09-14T07:37:43.954 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.48908.log: -- replaced with /var/log/ceph/cluster2-client.mirror.106079.log.gz 2024-09-14T07:37:43.955 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63239.log 2024-09-14T07:37:43.955 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61530.log.gz 2024-09-14T07:37:43.955 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59102.log 2024-09-14T07:37:43.955 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68206.log.gz 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125323.log 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76444.log.gz 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59102.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49749.log.gz 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49784.log.gz 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49842.log.gz 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50772.log.gz 2024-09-14T07:37:43.956 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50958.log.gz 2024-09-14T07:37:43.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51423.log.gz 2024-09-14T07:37:43.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65540.log.gz 2024-09-14T07:37:43.957 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81786.log.gz 2024-09-14T07:37:43.960 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.125323.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.125323.log.gz 2024-09-14T07:37:43.960 INFO:teuthology.orchestra.run.smithi169.stderr: 96.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75157.log.gz 2024-09-14T07:37:43.960 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53738.log 2024-09-14T07:37:43.960 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62030.log.gz 2024-09-14T07:37:43.960 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48071.log 2024-09-14T07:37:43.961 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53738.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71537.log.gz 2024-09-14T07:37:43.962 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118945.log 2024-09-14T07:37:43.962 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59102.log.gz 2024-09-14T07:37:43.962 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.48071.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35047.log 2024-09-14T07:37:43.962 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31118.log 2024-09-14T07:37:43.962 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35047.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.118945.log.gz 2024-09-14T07:37:43.963 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72281.log.gz 2024-09-14T07:37:43.963 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72374.log.gz 2024-09-14T07:37:43.963 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41011.log.gz 2024-09-14T07:37:43.964 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47234.log.gz 2024-09-14T07:37:43.965 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54824.log.gz 2024-09-14T07:37:43.966 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55103.log.gz 2024-09-14T07:37:43.967 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63239.log.gz 2024-09-14T07:37:43.967 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63611.log.gz 2024-09-14T07:37:43.968 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38363.log 2024-09-14T07:37:43.968 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.31118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25429.log 2024-09-14T07:37:43.969 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105740.log 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.25429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30913.log 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25429.log.gz 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103036.log 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105740.log: /var/log/ceph/cluster2-client.admin.30913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100866.log 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30913.log.gz 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.103036.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105740.log.gz 2024-09-14T07:37:43.970 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61003.log.gz 2024-09-14T07:37:43.971 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40228.log.gz 2024-09-14T07:37:43.971 INFO:teuthology.orchestra.run.smithi169.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31118.log.gz 2024-09-14T07:37:43.972 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95878.log 2024-09-14T07:37:43.972 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51737.log.gz 2024-09-14T07:37:43.972 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81281.log.gz 2024-09-14T07:37:43.973 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100866.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35849.log.gz 2024-09-14T07:37:43.973 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100866.log.gz 2024-09-14T07:37:43.973 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36593.log.gz 2024-09-14T07:37:43.973 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92707.log 2024-09-14T07:37:43.973 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41476.log.gz 2024-09-14T07:37:43.974 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43096.log.gz 2024-09-14T07:37:43.975 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.45051.log.gz 2024-09-14T07:37:43.975 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45298.log.gz 2024-09-14T07:37:43.976 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.95878.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95878.log.gz 2024-09-14T07:37:43.976 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66098.log 2024-09-14T07:37:43.976 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83538.log 2024-09-14T07:37:43.976 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.92707.log: /var/log/ceph/cluster2-client.mirror.66098.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92707.log.gz 2024-09-14T07:37:43.976 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80508.log 2024-09-14T07:37:43.976 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48071.log.gz 2024-09-14T07:37:43.979 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.83538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63297.log 2024-09-14T07:37:43.979 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54208.log 2024-09-14T07:37:43.980 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80609.log 2024-09-14T07:37:43.980 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.63297.log: /var/log/ceph/cluster2-client.mirror.54208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49001.log 2024-09-14T07:37:43.982 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.80609.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128213.log 2024-09-14T07:37:43.982 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30130.log 2024-09-14T07:37:43.982 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77781.log 2024-09-14T07:37:43.982 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.30130.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128213.log.gz 2024-09-14T07:37:43.983 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30130.log.gz 2024-09-14T07:37:43.983 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131545.log 2024-09-14T07:37:43.983 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.77781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59572.log 2024-09-14T07:37:43.984 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.131545.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.131545.log.gz 2024-09-14T07:37:43.984 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89677.log 2024-09-14T07:37:43.985 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.59572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29720.log 2024-09-14T07:37:43.985 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105841.log 2024-09-14T07:37:43.985 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120473.log 2024-09-14T07:37:43.986 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.105841.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105841.log.gz 2024-09-14T07:37:43.987 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41197.log 2024-09-14T07:37:43.988 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120473.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.120473.log.gz 2024-09-14T07:37:43.988 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35291.log 2024-09-14T07:37:43.988 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64413.log.gz 2024-09-14T07:37:43.989 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67185.log.gz 2024-09-14T07:37:43.989 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41197.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127313.log 2024-09-14T07:37:43.990 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38921.log 2024-09-14T07:37:43.991 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127313.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127313.log.gz 2024-09-14T07:37:43.991 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75603.log.gz 2024-09-14T07:37:43.993 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93047.log 2024-09-14T07:37:43.994 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.38921.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128393.log 2024-09-14T07:37:43.994 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.93047.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77781.log.gz 2024-09-14T07:37:43.995 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.93047.log.gz 2024-09-14T07:37:43.996 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84816.log.gz 2024-09-14T07:37:43.996 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80508.log.gz 2024-09-14T07:37:43.996 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113116.log 2024-09-14T07:37:43.997 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128393.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.128393.log.gz 2024-09-14T07:37:43.997 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40527.log 2024-09-14T07:37:43.998 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37000.log.gz 2024-09-14T07:37:43.998 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113116.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.113116.log.gz 2024-09-14T07:37:43.999 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97769.log 2024-09-14T07:37:43.999 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.40527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129424.log 2024-09-14T07:37:44.000 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.97769.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.97769.log.gz 2024-09-14T07:37:44.000 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96730.log 2024-09-14T07:37:44.001 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.129424.log: 54.4% -- replaced with /var/log/ceph/cluster2-client.mirror.129424.log.gz 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85788.log.gz 2024-09-14T07:37:44.001 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:44.003 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74825.log.gz 2024-09-14T07:37:44.003 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74922.log.gz 2024-09-14T07:37:44.004 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34023.log 2024-09-14T07:37:44.005 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.96730.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63762.log.gz 2024-09-14T07:37:44.005 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63797.log.gz 2024-09-14T07:37:44.006 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64134.log.gz 2024-09-14T07:37:44.006 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96730.log.gz 2024-09-14T07:37:44.006 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85830.log 2024-09-14T07:37:44.007 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34023.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53160.log 2024-09-14T07:37:44.008 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85830.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40558.log 2024-09-14T07:37:44.009 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.53160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54080.log 2024-09-14T07:37:44.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.admin.40558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133749.log 2024-09-14T07:37:44.010 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54080.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123773.log 2024-09-14T07:37:44.011 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133749.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.133749.log.gz 2024-09-14T07:37:44.013 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102285.log 2024-09-14T07:37:44.013 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.123773.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.68411.log.gz 2024-09-14T07:37:44.015 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.123773.log.gz 2024-09-14T07:37:44.015 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133929.log 2024-09-14T07:37:44.018 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.102285.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45552.log.gz 2024-09-14T07:37:44.018 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49226.log.gz 2024-09-14T07:37:44.022 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50214.log.gz 2024-09-14T07:37:44.022 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66098.log.gz 2024-09-14T07:37:44.022 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.71479.log.gz 2024-09-14T07:37:44.022 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40860.log.gz 2024-09-14T07:37:44.022 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47083.log.gz 2024-09-14T07:37:44.023 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53160.log.gz 2024-09-14T07:37:44.024 INFO:teuthology.orchestra.run.smithi169.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.102285.log.gz 2024-09-14T07:37:44.024 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.116885.log.gz 2024-09-14T07:37:44.025 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120279.log 2024-09-14T07:37:44.025 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.133929.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132682.log 2024-09-14T07:37:44.026 INFO:teuthology.orchestra.run.smithi169.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.133929.log.gz 2024-09-14T07:37:44.026 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.120279.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101706.log 2024-09-14T07:37:44.027 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2%/var/log/ceph/cluster1-client.mirror.132682.log: -- replaced with /var/log/ceph/cluster2-client.mirror.120279.log.gz 2024-09-14T07:37:44.027 INFO:teuthology.orchestra.run.smithi169.stderr: 84.6% -- replaced with /var/log/ceph/cluster1-client.mirror.132682.log.gz 2024-09-14T07:37:44.027 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41232.log 2024-09-14T07:37:44.028 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.101706.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85336.log.gz 2024-09-14T07:37:44.029 INFO:teuthology.orchestra.run.smithi169.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101706.log.gz 2024-09-14T07:37:44.029 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40459.log 2024-09-14T07:37:44.029 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32310.log.gz 2024-09-14T07:37:44.030 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41232.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99085.log 2024-09-14T07:37:44.030 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34023.log.gz 2024-09-14T07:37:44.030 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33988.log.gz 2024-09-14T07:37:44.031 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.40459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88263.log 2024-09-14T07:37:44.032 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85508.log 2024-09-14T07:37:44.032 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.99085.log.gz 2024-09-14T07:37:44.038 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88263.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38363.log.gz 2024-09-14T07:37:44.039 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38921.log.gz 2024-09-14T07:37:44.039 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52109.log.gz 2024-09-14T07:37:44.039 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109335.log 2024-09-14T07:37:44.040 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.85508.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52616.log.gz 2024-09-14T07:37:44.040 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61375.log 2024-09-14T07:37:44.041 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.109335.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.109335.log.gz 2024-09-14T07:37:44.041 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88399.log 2024-09-14T07:37:44.042 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.61375.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35791.log.gz 2024-09-14T07:37:44.042 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79358.log 2024-09-14T07:37:44.042 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.88399.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43340.log.gz 2024-09-14T07:37:44.043 INFO:teuthology.orchestra.run.smithi169.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88399.log.gz 2024-09-14T07:37:44.044 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56312.log 2024-09-14T07:37:44.044 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79358.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79797.log 2024-09-14T07:37:44.045 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.56312.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85620.log 2024-09-14T07:37:44.046 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.79797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122714.log 2024-09-14T07:37:44.049 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.85620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54452.log 2024-09-14T07:37:44.050 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.122714.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122714.log.gz 2024-09-14T07:37:44.050 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113631.log 2024-09-14T07:37:44.051 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.54452.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39239.log 2024-09-14T07:37:44.052 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.113631.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69863.log.gz 2024-09-14T07:37:44.052 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113631.log.gz 2024-09-14T07:37:44.052 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77839.log.gz 2024-09-14T07:37:44.055 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76351.log 2024-09-14T07:37:44.055 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.39239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67499.log 2024-09-14T07:37:44.056 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.53561.log 2024-09-14T07:37:44.056 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67499.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91958.log 2024-09-14T07:37:44.066 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.2.53561.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48908.log.gz 2024-09-14T07:37:44.066 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80166.log.gz 2024-09-14T07:37:44.070 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72746.log.gz 2024-09-14T07:37:44.075 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76851.log 2024-09-14T07:37:44.075 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.40558.log.gz 2024-09-14T07:37:44.075 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.91958.log: 3.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91958.log.gz 2024-09-14T07:37:44.086 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71572.log 2024-09-14T07:37:44.095 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.76851.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.1.72865.log.gz 2024-09-14T07:37:44.100 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52708.log 2024-09-14T07:37:44.109 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.71572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35942.log 2024-09-14T07:37:44.110 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52708.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.98901.log.gz 2024-09-14T07:37:44.110 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115038.log 2024-09-14T07:37:44.111 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37805.log 2024-09-14T07:37:44.112 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.115038.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.115038.log.gz 2024-09-14T07:37:44.112 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46862.log 2024-09-14T07:37:44.113 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.37805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100587.log 2024-09-14T07:37:44.114 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89563.log 2024-09-14T07:37:44.115 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.100587.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.100587.log.gz 2024-09-14T07:37:44.115 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78585.log.gz 2024-09-14T07:37:44.116 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65505.log 2024-09-14T07:37:44.116 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.89563.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79292.log.gz 2024-09-14T07:37:44.117 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89563.log.gz 2024-09-14T07:37:44.117 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2024-09-14T07:37:44.118 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.65505.log: 92.7%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.45572.log.gz 2024-09-14T07:37:44.118 INFO:teuthology.orchestra.run.smithi169.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81382.log 2024-09-14T07:37:44.118 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94773.log 2024-09-14T07:37:44.119 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.81382.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53539.log.gz 2024-09-14T07:37:44.119 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77037.log.gz 2024-09-14T07:37:44.120 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53673.log.gz 2024-09-14T07:37:44.120 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65505.log.gz 2024-09-14T07:37:44.120 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65970.log.gz 2024-09-14T07:37:44.122 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46990.log.gz 2024-09-14T07:37:44.122 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71572.log.gz 2024-09-14T07:37:44.122 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54673.log.gz 2024-09-14T07:37:44.123 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99925.log 2024-09-14T07:37:44.125 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.94773.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.94773.log.gz 2024-09-14T07:37:44.126 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118732.log 2024-09-14T07:37:44.126 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.99925.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85830.log.gz 2024-09-14T07:37:44.127 INFO:teuthology.orchestra.run.smithi169.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99925.log.gz 2024-09-14T07:37:44.128 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129000.log 2024-09-14T07:37:44.128 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118732.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118732.log.gz 2024-09-14T07:37:44.129 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95512.log 2024-09-14T07:37:44.132 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.129000.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32775.log.gz 2024-09-14T07:37:44.133 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61061.log.gz 2024-09-14T07:37:44.133 INFO:teuthology.orchestra.run.smithi169.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.129000.log.gz 2024-09-14T07:37:44.133 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34919.log 2024-09-14T07:37:44.135 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.95512.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.95512.log.gz 2024-09-14T07:37:44.135 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112563.log 2024-09-14T07:37:44.136 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.34919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.77979.log 2024-09-14T07:37:44.137 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.112563.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112563.log.gz 2024-09-14T07:37:44.142 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70421.log 2024-09-14T07:37:44.142 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.2.77979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97341.log 2024-09-14T07:37:44.143 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.89395.log.gz 2024-09-14T07:37:44.143 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.70421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121502.log 2024-09-14T07:37:44.144 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49036.log 2024-09-14T07:37:44.144 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.121502.log: /var/log/ceph/cluster2-client.mirror.97341.log: 87.9% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97341.log.gz 2024-09-14T07:37:44.144 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.121502.log.gz 2024-09-14T07:37:44.144 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46304.log 2024-09-14T07:37:44.145 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.49036.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62867.log 2024-09-14T07:37:44.145 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.34919.log.gz 2024-09-14T07:37:44.146 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.46304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35384.log 2024-09-14T07:37:44.147 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120936.log 2024-09-14T07:37:44.147 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58265.log.gz 2024-09-14T07:37:44.147 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.35384.log: 88.2%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.37805.log.gz -5 2024-09-14T07:37:44.147 INFO:teuthology.orchestra.run.smithi169.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.107337.log 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127245.log 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.120936.log: /var/log/ceph/cluster2-client.mirror.107337.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.120936.log.gz 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38398.log.gz 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39239.log.gz 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr: 81.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30480.log.gz 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107337.log.gz 2024-09-14T07:37:44.149 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64692.log.gz 2024-09-14T07:37:44.152 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62495.log 2024-09-14T07:37:44.152 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126420.log 2024-09-14T07:37:44.153 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.127245.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127245.log.gz 2024-09-14T07:37:44.153 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128483.log 2024-09-14T07:37:44.154 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126420.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126420.log.gz 2024-09-14T07:37:44.155 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.62495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131900.log 2024-09-14T07:37:44.156 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.128483.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.128483.log.gz 2024-09-14T07:37:44.156 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68499.log 2024-09-14T07:37:44.156 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67127.log 2024-09-14T07:37:44.157 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster1-client.mirror.131900.log: /var/log/ceph/cluster2-client.mirror.68499.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.131900.log.gz 2024-09-14T07:37:44.157 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124490.log 2024-09-14T07:37:44.157 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.134197.log 2024-09-14T07:37:44.159 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.124490.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44735.log.gz 2024-09-14T07:37:44.159 INFO:teuthology.orchestra.run.smithi169.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.124490.log.gz 2024-09-14T07:37:44.160 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.67127.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127943.log 2024-09-14T07:37:44.161 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29670.log 2024-09-14T07:37:44.161 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.134197.log: /var/log/ceph/cluster1-client.mirror.127943.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.134197.log.gz 2024-09-14T07:37:44.161 INFO:teuthology.orchestra.run.smithi169.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.127943.log.gz 2024-09-14T07:37:44.161 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48478.log.gz 2024-09-14T07:37:44.163 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82392.log.gz 2024-09-14T07:37:44.163 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57649.log 2024-09-14T07:37:44.164 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95759.log 2024-09-14T07:37:44.165 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29670.log: /var/log/ceph/cluster2-client.mirror.57649.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29670.log.gz 2024-09-14T07:37:44.165 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126327.log 2024-09-14T07:37:44.165 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0%/var/log/ceph/cluster2-client.mirror.95759.log: -- replaced with /var/log/ceph/cluster2-client.mirror.76351.log.gz 2024-09-14T07:37:44.166 INFO:teuthology.orchestra.run.smithi169.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95759.log.gz 2024-09-14T07:37:44.166 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69805.log.gz 2024-09-14T07:37:44.167 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54917.log 2024-09-14T07:37:44.167 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29695.log 2024-09-14T07:37:44.168 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.126327.log: /var/log/ceph/cluster2-client.mirror.54917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118829.log 2024-09-14T07:37:44.168 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.admin.29695.log: 0.0% 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.126327.log.gz 2024-09-14T07:37:44.169 INFO:teuthology.orchestra.run.smithi169.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.29695.log.gz 2024-09-14T07:37:44.169 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102850.log 2024-09-14T07:37:44.170 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86862.log 2024-09-14T07:37:44.170 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.118829.log: /var/log/ceph/cluster2-client.mirror.102850.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118829.log.gz 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.102850.log.gz 2024-09-14T07:37:44.170 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:44.170 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40767.log 2024-09-14T07:37:44.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.86862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41290.log 2024-09-14T07:37:44.171 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.40767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78853.log 2024-09-14T07:37:44.172 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.41290.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52202.log 2024-09-14T07:37:44.173 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.78853.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54301.log 2024-09-14T07:37:44.174 INFO:teuthology.orchestra.run.smithi169.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73116.log 2024-09-14T07:37:44.177 INFO:teuthology.orchestra.run.smithi169.stderr:/var/log/ceph/cluster2-client.mirror.52202.log: /var/log/ceph/cluster2-client.mirror.54301.log: /var/log/ceph/cluster2-client.mirror.73116.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39669.log.gz 2024-09-14T07:37:44.186 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61375.log.gz 2024-09-14T07:37:44.187 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.65429.log.gz 2024-09-14T07:37:44.188 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83538.log.gz 2024-09-14T07:37:44.188 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83604.log.gz 2024-09-14T07:37:44.190 INFO:teuthology.orchestra.run.smithi169.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86816.log.gz 2024-09-14T07:37:44.262 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54080.log.gz 2024-09-14T07:37:44.263 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75115.log.gz 2024-09-14T07:37:44.263 INFO:teuthology.orchestra.run.smithi169.stderr: 74.1% -- replaced with /var/log/ceph/cluster1-client.mirror.68620.log.gz 2024-09-14T07:37:44.263 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76851.log.gz 2024-09-14T07:37:44.264 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90432.log.gz 2024-09-14T07:37:44.274 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103036.log.gz 2024-09-14T07:37:44.276 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58393.log.gz 2024-09-14T07:37:44.277 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62495.log.gz 2024-09-14T07:37:44.277 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71386.log.gz 2024-09-14T07:37:44.277 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40767.log.gz 2024-09-14T07:37:44.278 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.47141.log.gz 2024-09-14T07:37:44.279 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54917.log.gz 2024-09-14T07:37:44.290 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89677.log.gz 2024-09-14T07:37:44.302 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53650.log.gz 2024-09-14T07:37:44.313 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37747.log.gz 2024-09-14T07:37:44.314 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53715.log.gz 2024-09-14T07:37:44.314 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38863.log.gz 2024-09-14T07:37:44.314 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39890.log.gz 2024-09-14T07:37:44.316 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42782.log.gz 2024-09-14T07:37:44.316 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43805.log.gz 2024-09-14T07:37:44.316 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44026.log.gz 2024-09-14T07:37:44.316 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44456.log.gz 2024-09-14T07:37:44.322 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.90567.log.gz 2024-09-14T07:37:44.332 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88263.log.gz 2024-09-14T07:37:44.334 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69677.log.gz 2024-09-14T07:37:44.335 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77467.log.gz 2024-09-14T07:37:44.337 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73430.log.gz 2024-09-14T07:37:44.338 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78954.log.gz 2024-09-14T07:37:44.339 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82998.log.gz 2024-09-14T07:37:44.341 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49036.log.gz 2024-09-14T07:37:44.344 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84730.log.gz 2024-09-14T07:37:44.372 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85078.log.gz 2024-09-14T07:37:44.389 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52756.log.gz 2024-09-14T07:37:44.397 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53693.log.gz 2024-09-14T07:37:44.416 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47327.log.gz 2024-09-14T07:37:44.416 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61658.log.gz 2024-09-14T07:37:44.417 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69374.log.gz 2024-09-14T07:37:44.420 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85999.log.gz 2024-09-14T07:37:44.425 INFO:teuthology.orchestra.run.smithi169.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85535.log.gz 2024-09-14T07:37:44.426 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39611.log.gz 2024-09-14T07:37:44.427 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.85555.log.gz 2024-09-14T07:37:44.428 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52016.log.gz 2024-09-14T07:37:44.429 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81013.log.gz 2024-09-14T07:37:44.429 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52651.log.gz 2024-09-14T07:37:44.432 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.36779.log.gz 2024-09-14T07:37:44.434 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41197.log.gz 2024-09-14T07:37:44.435 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81988.log.gz 2024-09-14T07:37:44.437 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.46490.log.gz 2024-09-14T07:37:44.438 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46525.log.gz 2024-09-14T07:37:44.438 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.46676.log.gz 2024-09-14T07:37:44.438 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46711.log.gz 2024-09-14T07:37:44.439 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67499.log.gz 2024-09-14T07:37:44.441 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75975.log.gz 2024-09-14T07:37:44.441 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56161.log.gz 2024-09-14T07:37:44.441 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74546.log.gz 2024-09-14T07:37:44.443 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80609.log.gz 2024-09-14T07:37:44.448 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.60014.log.gz 2024-09-14T07:37:44.459 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85683.log.gz 2024-09-14T07:37:44.469 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84223.log.gz 2024-09-14T07:37:44.502 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54208.log.gz 2024-09-14T07:37:44.502 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46862.log.gz 2024-09-14T07:37:44.503 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54452.log.gz 2024-09-14T07:37:44.513 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48943.log.gz 2024-09-14T07:37:44.514 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49563.log.gz 2024-09-14T07:37:44.514 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49001.log.gz 2024-09-14T07:37:44.514 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51179.log.gz 2024-09-14T07:37:44.514 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47606.log.gz 2024-09-14T07:37:44.514 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58823.log.gz 2024-09-14T07:37:44.515 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53391.log.gz 2024-09-14T07:37:44.531 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.85508.log.gz 2024-09-14T07:37:44.533 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56533.log.gz 2024-09-14T07:37:44.535 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68113.log.gz 2024-09-14T07:37:44.535 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.69129.log.gz 2024-09-14T07:37:44.536 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.40527.log.gz 2024-09-14T07:37:44.538 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32868.log.gz 2024-09-14T07:37:44.539 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33368.log.gz 2024-09-14T07:37:44.540 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44177.log.gz 2024-09-14T07:37:44.541 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64506.log.gz 2024-09-14T07:37:44.553 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74697.log.gz 2024-09-14T07:37:44.553 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78853.log.gz 2024-09-14T07:37:44.566 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59826.log.gz 2024-09-14T07:37:44.588 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59572.log.gz 2024-09-14T07:37:44.591 INFO:teuthology.orchestra.run.smithi169.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85620.log.gz 2024-09-14T07:37:44.616 INFO:teuthology.orchestra.run.smithi169.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.29720.log.gz 2024-09-14T07:37:44.617 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50156.log.gz 2024-09-14T07:37:44.619 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51144.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.51086.log.gz 2024-09-14T07:37:44.619 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:44.620 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63297.log.gz 2024-09-14T07:37:44.747 INFO:teuthology.orchestra.run.smithi169.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84925.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% 57.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30862.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.84156.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.85465.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38770.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73337.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64262.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35942.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57649.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35291.log.gz 2024-09-14T07:37:44.751 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46304.log.gz 2024-09-14T07:37:44.752 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57835.log.gz 2024-09-14T07:37:44.752 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.36349.log.gz 2024-09-14T07:37:44.752 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76944.log.gz 2024-09-14T07:37:44.776 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.0.87132.log.gz 2024-09-14T07:37:44.798 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79358.log.gz 2024-09-14T07:37:44.798 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79797.log.gz 2024-09-14T07:37:44.799 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56312.log.gz 2024-09-14T07:37:44.848 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78484.log.gz 2024-09-14T07:37:44.851 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62867.log.gz 2024-09-14T07:37:44.865 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84839.log.gz 2024-09-14T07:37:44.868 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84248.log.gz 2024-09-14T07:37:44.904 INFO:teuthology.orchestra.run.smithi169.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40481.log.gz 2024-09-14T07:37:44.904 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68434.log.gz 2024-09-14T07:37:44.927 INFO:teuthology.orchestra.run.smithi169.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52728.log.gz 2024-09-14T07:37:44.933 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68454.log.gz 2024-09-14T07:37:44.959 INFO:teuthology.orchestra.run.smithi169.stderr: 97.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52798.log.gz 2024-09-14T07:37:44.969 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65199.log.gz 2024-09-14T07:37:45.018 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35384.log.gz 2024-09-14T07:37:45.018 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.40414.log.gz 2024-09-14T07:37:45.019 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51888.log.gz 2024-09-14T07:37:45.019 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52202.log.gz 2024-09-14T07:37:45.019 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42910.log.gz 2024-09-14T07:37:45.020 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57428.log.gz 2024-09-14T07:37:45.020 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79156.log.gz 2024-09-14T07:37:45.020 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67127.log.gz 2024-09-14T07:37:45.020 INFO:teuthology.orchestra.run.smithi169.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77874.log.gz 2024-09-14T07:37:45.021 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73244.log.gz 2024-09-14T07:37:45.021 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75824.log.gz 2024-09-14T07:37:45.021 INFO:teuthology.orchestra.run.smithi169.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85641.log.gz 2024-09-14T07:37:45.063 INFO:teuthology.orchestra.run.smithi169.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70421.log.gz 2024-09-14T07:37:45.080 INFO:teuthology.orchestra.run.smithi169.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.admin.30982.log.gz 2024-09-14T07:37:45.082 INFO:teuthology.orchestra.run.smithi169.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48013.log.gz 2024-09-14T07:37:45.115 INFO:teuthology.orchestra.run.smithi169.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.75092.log.gz 2024-09-14T07:37:45.119 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73116.log.gz 2024-09-14T07:37:45.120 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68476.log.gz 2024-09-14T07:37:45.121 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.85058.log.gz 2024-09-14T07:37:45.121 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.35047.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.44828.log.gz 2024-09-14T07:37:45.121 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:37:45.122 INFO:teuthology.orchestra.run.smithi169.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85273.log.gz 2024-09-14T07:37:45.126 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69991.log.gz 2024-09-14T07:37:45.148 INFO:teuthology.orchestra.run.smithi169.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58358.log.gz 2024-09-14T07:37:45.153 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80574.log.gz 2024-09-14T07:37:45.214 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85230.log.gz 2024-09-14T07:37:45.252 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39704.log.gz 2024-09-14T07:37:45.254 INFO:teuthology.orchestra.run.smithi169.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64785.log.gz 2024-09-14T07:37:45.268 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41290.log.gz 2024-09-14T07:37:45.355 INFO:teuthology.orchestra.run.smithi169.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.1.39946.log.gz 2024-09-14T07:37:45.361 INFO:teuthology.orchestra.run.smithi169.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56684.log.gz 2024-09-14T07:37:45.400 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.30505.log.gz 2024-09-14T07:37:45.455 INFO:teuthology.orchestra.run.smithi169.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52481.log.gz 2024-09-14T07:37:45.463 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41232.log.gz 2024-09-14T07:37:45.469 INFO:teuthology.orchestra.run.smithi169.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77374.log.gz 2024-09-14T07:37:45.495 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72653.log.gz 2024-09-14T07:37:45.501 INFO:teuthology.orchestra.run.smithi169.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34453.log.gz 2024-09-14T07:37:45.506 INFO:teuthology.orchestra.run.smithi169.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86862.log.gz 2024-09-14T07:37:45.574 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84415.log.gz 2024-09-14T07:37:45.584 INFO:teuthology.orchestra.run.smithi169.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81382.log.gz 2024-09-14T07:37:45.689 INFO:teuthology.orchestra.run.smithi169.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.77979.log.gz 2024-09-14T07:37:46.263 INFO:teuthology.orchestra.run.smithi169.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51330.log.gz 2024-09-14T07:37:46.264 INFO:teuthology.orchestra.run.smithi169.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65633.log.gz 2024-09-14T07:37:46.417 INFO:teuthology.orchestra.run.smithi169.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.53425.log.gz 2024-09-14T07:37:46.632 INFO:teuthology.orchestra.run.smithi169.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69217.log.gz 2024-09-14T07:37:46.639 INFO:teuthology.orchestra.run.smithi169.stderr: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69239.log.gz 2024-09-14T07:37:46.948 INFO:teuthology.orchestra.run.smithi169.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54301.log.gz 2024-09-14T07:37:48.344 INFO:teuthology.orchestra.run.smithi169.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52708.log.gz 2024-09-14T07:37:49.943 INFO:teuthology.orchestra.run.smithi169.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65241.log.gz 2024-09-14T07:37:49.951 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40459.log.gz 2024-09-14T07:37:49.957 INFO:teuthology.orchestra.run.smithi169.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45509.log.gz 2024-09-14T07:37:49.965 INFO:teuthology.orchestra.run.smithi169.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.45594.log.gz 2024-09-14T07:37:51.353 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87376.log.gz 2024-09-14T07:37:52.199 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87598.log.gz 2024-09-14T07:37:56.537 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.67927.log.gz 2024-09-14T07:37:57.009 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65457.log.gz 2024-09-14T07:37:57.322 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52821.log.gz 2024-09-14T07:37:57.830 INFO:teuthology.orchestra.run.smithi169.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.78079.log.gz 2024-09-14T07:37:57.925 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68499.log.gz 2024-09-14T07:37:59.491 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86570.log.gz 2024-09-14T07:38:00.309 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40586.log.gz 2024-09-14T07:38:01.702 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87252.log.gz 2024-09-14T07:38:01.718 INFO:teuthology.orchestra.run.smithi169.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86120.log.gz 2024-09-14T07:38:01.770 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87425.log.gz 2024-09-14T07:38:01.868 INFO:teuthology.orchestra.run.smithi169.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87316.log.gz 2024-09-14T07:38:02.029 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59992.log.gz 2024-09-14T07:38:02.602 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68598.log.gz 2024-09-14T07:38:03.111 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52907.log.gz 2024-09-14T07:38:03.337 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86239.log.gz 2024-09-14T07:38:03.533 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.53561.log.gz 2024-09-14T07:38:03.618 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87019.log.gz 2024-09-14T07:38:04.055 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75276.log.gz 2024-09-14T07:38:04.289 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69352.log.gz 2024-09-14T07:38:05.085 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86839.log.gz 2024-09-14T07:38:06.489 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86650.log.gz 2024-09-14T07:38:07.235 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53935.log.gz 2024-09-14T07:38:07.337 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45702.log.gz 2024-09-14T07:38:08.239 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40504.log.gz 2024-09-14T07:38:09.000 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45617.log.gz 2024-09-14T07:38:09.109 INFO:teuthology.orchestra.run.smithi169.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59891.log.gz 2024-09-14T07:38:09.263 INFO:teuthology.orchestra.run.smithi169.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65264.log.gz 2024-09-14T07:38:09.465 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86343.log.gz 2024-09-14T07:38:09.931 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53738.log.gz 2024-09-14T07:38:10.454 INFO:teuthology.orchestra.run.smithi169.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87526.log.gz 2024-09-14T07:38:10.586 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86410.log.gz 2024-09-14T07:38:12.345 INFO:teuthology.orchestra.run.smithi169.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69262.log.gz 2024-09-14T07:38:13.579 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31811.log.gz 2024-09-14T07:38:14.924 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.86862.log.gz 2024-09-14T07:38:16.386 INFO:teuthology.orchestra.run.smithi169.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31983.log.gz 2024-09-14T07:38:16.996 INFO:teuthology.orchestra.run.smithi169.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75180.log.gz 2024-09-14T07:38:17.673 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.84415.log.gz 2024-09-14T07:38:21.408 INFO:teuthology.orchestra.run.smithi169.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.67927.log.gz 2024-09-14T07:38:24.653 INFO:teuthology.orchestra.run.smithi169.stderr: 91.0% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2024-09-14T07:38:28.521 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.69374.log.gz 2024-09-14T07:39:02.457 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.30405.log.gz 2024-09-14T07:39:22.782 INFO:teuthology.orchestra.run.smithi129.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2024-09-14T07:39:27.009 INFO:teuthology.orchestra.run.smithi129.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2024-09-14T07:39:37.224 INFO:teuthology.orchestra.run.smithi129.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2024-09-14T07:39:37.226 INFO:teuthology.orchestra.run.smithi129.stderr: 2024-09-14T07:39:37.226 INFO:teuthology.orchestra.run.smithi129.stderr:real 1m56.474s 2024-09-14T07:39:37.226 INFO:teuthology.orchestra.run.smithi129.stderr:user 3m44.727s 2024-09-14T07:39:37.226 INFO:teuthology.orchestra.run.smithi129.stderr:sys 0m9.895s 2024-09-14T07:42:20.999 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2024-09-14T07:42:48.235 INFO:teuthology.orchestra.run.smithi169.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2024-09-14T07:43:15.687 INFO:teuthology.orchestra.run.smithi169.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2024-09-14T07:43:15.687 INFO:teuthology.orchestra.run.smithi169.stderr: 2024-09-14T07:43:15.688 INFO:teuthology.orchestra.run.smithi169.stderr:real 5m34.920s 2024-09-14T07:43:15.688 INFO:teuthology.orchestra.run.smithi169.stderr:user 7m17.052s 2024-09-14T07:43:15.688 INFO:teuthology.orchestra.run.smithi169.stderr:sys 0m22.611s 2024-09-14T07:43:15.689 INFO:tasks.ceph:Archiving logs... 2024-09-14T07:43:15.690 DEBUG:teuthology.misc:Transferring archived files from smithi129:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/remote/smithi129/log 2024-09-14T07:43:15.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T07:43:37.921 DEBUG:teuthology.misc:Transferring archived files from smithi169:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/remote/smithi169/log 2024-09-14T07:43:37.922 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T07:44:11.403 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T07:44:11.433 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T07:44:11.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T07:44:11.436 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T07:44:11.500 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T07:44:11.514 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T07:44:11.514 DEBUG:teuthology.orchestra.run.smithi129:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T07:44:11.517 DEBUG:teuthology.orchestra.run.smithi169:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T07:44:11.564 INFO:teuthology.orchestra.run.smithi129.stdout: remote refid st t when poll reach delay offset jitter 2024-09-14T07:44:11.565 INFO:teuthology.orchestra.run.smithi129.stdout:============================================================================== 2024-09-14T07:44:11.565 INFO:teuthology.orchestra.run.smithi129.stdout:*hv01.front.sepi 67.205.162.81 3 u 40 64 377 0.094 -1.936 0.350 2024-09-14T07:44:11.565 INFO:teuthology.orchestra.run.smithi129.stdout:+hv02.front.sepi 63.231.80.2 3 u 18 64 377 0.105 -3.127 0.168 2024-09-14T07:44:11.565 INFO:teuthology.orchestra.run.smithi129.stdout:+hv03.front.sepi 144.34.193.110 3 u 50 64 377 0.079 -1.162 0.447 2024-09-14T07:44:11.565 INFO:teuthology.orchestra.run.smithi129.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-14T07:44:11.649 INFO:teuthology.orchestra.run.smithi169.stdout: remote refid st t when poll reach delay offset jitter 2024-09-14T07:44:11.649 INFO:teuthology.orchestra.run.smithi169.stdout:============================================================================== 2024-09-14T07:44:11.649 INFO:teuthology.orchestra.run.smithi169.stdout:*hv01.front.sepi 67.205.162.81 3 u 20 64 377 0.136 -0.418 0.196 2024-09-14T07:44:11.649 INFO:teuthology.orchestra.run.smithi169.stdout:+hv02.front.sepi 63.231.80.2 3 u 44 64 377 0.091 -1.938 0.201 2024-09-14T07:44:11.649 INFO:teuthology.orchestra.run.smithi169.stdout:+hv03.front.sepi 144.34.193.110 3 u 47 64 377 0.099 -0.002 0.598 2024-09-14T07:44:11.649 INFO:teuthology.orchestra.run.smithi169.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-14T07:44:11.651 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T07:44:11.665 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T07:44:11.666 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T07:44:11.695 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T07:44:11.725 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T07:44:11.757 INFO:teuthology.task.internal:Duration was 3770.916797 seconds 2024-09-14T07:44:11.758 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T07:44:11.792 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T07:44:11.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T07:44:11.795 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T07:44:12.026 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T07:44:12.026 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T07:44:12.026 DEBUG:teuthology.orchestra.run.smithi129:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T07:44:12.037 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi169.front.sepia.ceph.com 2024-09-14T07:44:12.038 DEBUG:teuthology.orchestra.run.smithi169:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T07:44:12.048 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T07:44:12.048 DEBUG:teuthology.orchestra.run.smithi129:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T07:44:12.082 DEBUG:teuthology.orchestra.run.smithi169:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T07:44:12.122 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T07:44:12.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T07:44:12.166 DEBUG:teuthology.orchestra.run.smithi169:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T07:44:12.356 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T07:44:12.368 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T07:44:12.368 DEBUG:teuthology.orchestra.run.smithi129:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T07:44:12.394 DEBUG:teuthology.orchestra.run.smithi169:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T07:44:12.429 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T07:44:12.443 DEBUG:teuthology.orchestra.run.smithi129:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T07:44:12.445 DEBUG:teuthology.orchestra.run.smithi169:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T07:44:12.469 INFO:teuthology.orchestra.run.smithi129.stdout:kernel.core_pattern = core 2024-09-14T07:44:12.500 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern = core 2024-09-14T07:44:12.515 DEBUG:teuthology.orchestra.run.smithi129:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T07:44:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T07:44:12.553 DEBUG:teuthology.orchestra.run.smithi169:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T07:44:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T07:44:12.564 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T07:44:12.576 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T07:44:12.577 DEBUG:teuthology.misc:Transferring archived files from smithi129:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/remote/smithi129 2024-09-14T07:44:12.577 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T07:44:12.618 DEBUG:teuthology.misc:Transferring archived files from smithi169:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-10_20:16:17-rbd-main-distro-default-smithi/7900398/remote/smithi169 2024-09-14T07:44:12.618 DEBUG:teuthology.orchestra.run.smithi169:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T07:44:12.638 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T07:44:12.639 DEBUG:teuthology.orchestra.run.smithi129:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T07:44:12.658 DEBUG:teuthology.orchestra.run.smithi169:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T07:44:12.685 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T07:44:12.701 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T07:44:12.701 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T07:44:12.736 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T07:44:12.736 DEBUG:teuthology.orchestra.run.smithi129:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T07:44:12.739 DEBUG:teuthology.orchestra.run.smithi169:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T07:44:12.742 INFO:teuthology.orchestra.run.smithi129.stdout: 655429 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 07:44 /home/ubuntu/cephtest 2024-09-14T07:44:12.744 INFO:teuthology.orchestra.run.smithi169.stdout: 658669 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 07:44 /home/ubuntu/cephtest 2024-09-14T07:44:12.746 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T07:44:12.768 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T07:44:12.838 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-journal-stress-workunit} duration: 3770.916797399521 flavor: default owner: scheduled_teuthology@teuthology success: true 2024-09-14T07:44:12.838 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T07:44:12.923 INFO:teuthology.run:pass